Examples with SSTableWriter org.apache.cassandra.io.sstable.SSTableWriter used on opensource projects

Example 6 with SSTableWriter

use of org.apache.cassandra.io.sstable.SSTableWriter in project eiger by wlloyd.

the class CompactionTask method execute.

/**
     * For internal use and testing only.  The rest of the system should go through the submit* methods,
     * which are properly serialized.
     * Caller is in charge of marking/unmarking the sstables as compacting.
     */
public int execute(CompactionExecutorStatsCollector collector) throws IOException {
    // it is not empty, it may compact down to nothing if all rows are deleted.
    assert sstables != null;
    Set<SSTableReader> toCompact = new HashSet<SSTableReader>(sstables);
    if (!isCompactionInteresting(toCompact))
        return 0;
    // If use defined, we don't want to "trust" our space estimation. If
    // there isn't enough room, it's the user problem
    long expectedSize = isUserDefined ? 0 : cfs.getExpectedCompactedFileSize(toCompact);
    File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(expectedSize);
    if (partialCompactionsAcceptable()) {
        // Try again w/o the largest one.
        if (compactionFileLocation == null) {
            while (compactionFileLocation == null && toCompact.size() > 1) {
                logger.warn("insufficient space to compact all requested files " + StringUtils.join(toCompact, ", "));
                // Note that we have removed files that are still marked as compacting. This suboptimal but ok since the caller will unmark all
                // the sstables at the end.
                toCompact.remove(cfs.getMaxSizeFile(toCompact));
                compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
            }
        }
    }
    if (compactionFileLocation == null) {
        logger.warn("insufficient space to compact even the two smallest files, aborting");
        return 0;
    }
    if (DatabaseDescriptor.isSnapshotBeforeCompaction())
        cfs.table.snapshot(System.currentTimeMillis() + "-" + "compact-" + cfs.columnFamily);
    // sanity check: all sstables must belong to the same cfs
    for (SSTableReader sstable : toCompact) assert sstable.descriptor.cfname.equals(cfs.columnFamily);
    CompactionController controller = new CompactionController(cfs, toCompact, gcBefore, isUserDefined);
    // new sstables from flush can be added during a compaction, but only the compaction can remove them,
    // so in our single-threaded compaction world this is a valid way of determining if we're compacting
    // all the sstables (that existed when we started)
    logger.info("Compacting {}", toCompact);
    long startTime = System.currentTimeMillis();
    long totalkeysWritten = 0;
    long estimatedTotalKeys = Math.max(DatabaseDescriptor.getIndexInterval(), SSTableReader.getApproximateKeyCount(toCompact));
    long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(toCompact) / cfs.getCompactionStrategy().getMaxSSTableSize());
    long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
    if (logger.isDebugEnabled())
        logger.debug("Expected bloom filter size : " + keysPerSSTable);
    AbstractCompactionIterable ci = DatabaseDescriptor.isMultithreadedCompaction() ? new ParallelCompactionIterable(OperationType.COMPACTION, toCompact, controller) : new CompactionIterable(OperationType.COMPACTION, toCompact, controller);
    CloseableIterator<AbstractCompactedRow> iter = ci.iterator();
    Iterator<AbstractCompactedRow> nni = Iterators.filter(iter, Predicates.notNull());
    Map<DecoratedKey, Long> cachedKeys = new HashMap<DecoratedKey, Long>();
    // we can't preheat until the tracker has been set. This doesn't happen until we tell the cfs to
    // replace the old entries.  Track entries to preheat here until then.
    Map<SSTableReader, Map<DecoratedKey, Long>> cachedKeyMap = new HashMap<SSTableReader, Map<DecoratedKey, Long>>();
    Collection<SSTableReader> sstables = new ArrayList<SSTableReader>();
    Collection<SSTableWriter> writers = new ArrayList<SSTableWriter>();
    if (collector != null)
        collector.beginCompaction(ci);
    try {
        if (!nni.hasNext()) {
            // don't mark compacted in the finally block, since if there _is_ nondeleted data,
            // we need to sync it (via closeAndOpen) first, so there is no period during which
            // a crash could cause data loss.
            cfs.markCompacted(toCompact);
            return 0;
        }
        SSTableWriter writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
        writers.add(writer);
        while (nni.hasNext()) {
            if (ci.isStopped())
                throw new CompactionInterruptedException(ci.getCompactionInfo());
            AbstractCompactedRow row = nni.next();
            if (row.isEmpty())
                continue;
            long position = writer.append(row);
            totalkeysWritten++;
            if (DatabaseDescriptor.getPreheatKeyCache()) {
                for (SSTableReader sstable : toCompact) {
                    if (sstable.getCachedPosition(row.key, false) != null) {
                        cachedKeys.put(row.key, position);
                        break;
                    }
                }
            }
            if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer, position)) {
                SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact));
                cachedKeyMap.put(toIndex, cachedKeys);
                sstables.add(toIndex);
                if (nni.hasNext()) {
                    writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
                    writers.add(writer);
                    cachedKeys = new HashMap<DecoratedKey, Long>();
                }
            }
        }
    } catch (Exception e) {
        for (SSTableWriter writer : writers) writer.abort();
        throw FBUtilities.unchecked(e);
    } finally {
        iter.close();
        if (collector != null)
            collector.finishCompaction(ci);
    }
    cfs.replaceCompactedSSTables(toCompact, sstables);
    // TODO: this doesn't belong here, it should be part of the reader to load when the tracker is wired up
    for (Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet()) {
        SSTableReader key = ssTableReaderMapEntry.getKey();
        for (Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet()) key.cacheKey(entry.getKey(), entry.getValue());
    }
    long dTime = System.currentTimeMillis() - startTime;
    long startsize = SSTable.getTotalBytes(toCompact);
    long endsize = SSTable.getTotalBytes(sstables);
    double ratio = (double) endsize / (double) startsize;
    StringBuilder builder = new StringBuilder();
    builder.append("[");
    for (SSTableReader reader : sstables) builder.append(reader.getFilename()).append(",");
    builder.append("]");
    double mbps = dTime > 0 ? (double) endsize / (1024 * 1024) / ((double) dTime / 1000) : 0;
    logger.info(String.format("Compacted to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys at %fMB/s.  Time: %,dms.", builder.toString(), startsize, endsize, (int) (ratio * 100), totalkeysWritten, mbps, dTime));
    logger.debug(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
    return toCompact.size();
}

Example 7 with SSTableWriter

use of org.apache.cassandra.io.sstable.SSTableWriter in project eiger by wlloyd.

the class SSTableImport method importUnsorted.

private static int importUnsorted(JsonParser parser, ColumnFamily columnFamily, String ssTablePath, IPartitioner<?> partitioner) throws IOException {
    int importedKeys = 0;
    long start = System.currentTimeMillis();
    Map<?, ?> data = parser.readValueAs(new TypeReference<Map<?, ?>>() {
    });
    keyCountToImport = (keyCountToImport == null) ? data.size() : keyCountToImport;
    SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);
    System.out.printf("Importing %s keys...%n", keyCountToImport);
    // sort by dk representation, but hold onto the hex version
    SortedMap<DecoratedKey, String> decoratedKeys = new TreeMap<DecoratedKey, String>();
    for (Object keyObject : data.keySet()) {
        String key = (String) keyObject;
        decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)), key);
    }
    for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet()) {
        if (columnFamily.getType() == ColumnFamilyType.Super) {
            addToSuperCF((Map<?, ?>) data.get(rowKey.getValue()), columnFamily);
        } else {
            addToStandardCF((List<?>) data.get(rowKey.getValue()), columnFamily);
        }
        writer.append(rowKey.getKey(), columnFamily);
        columnFamily.clear();
        importedKeys++;
        long current = System.currentTimeMillis();
        if (// 5 secs.
        current - start >= 5000) {
            System.out.printf("Currently imported %d keys.%n", importedKeys);
            start = current;
        }
        if (keyCountToImport == importedKeys)
            break;
    }
    writer.closeAndOpenReader();
    return importedKeys;
}

Example 8 with SSTableWriter

use of org.apache.cassandra.io.sstable.SSTableWriter in project eiger by wlloyd.

the class SSTableExportTest method testRoundTripStandardCf.

@Test
public void testRoundTripStandardCf() throws IOException, ParseException {
    File tempSS = tempSSTableFile("Keyspace1", "Standard1");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    // Add rowA
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Standard1", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    // Import JSON to another SSTable file
    File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
    SSTableImport.importJson(tempJson.getPath(), "Keyspace1", "Standard1", tempSS2.getPath());
    reader = SSTableReader.open(Descriptor.fromFilename(tempSS2.getPath()));
    QueryFilter qf = QueryFilter.getNamesFilter(Util.dk("rowA"), new QueryPath("Standard1", null, null), ByteBufferUtil.bytes("name"));
    ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
    assertTrue(cf != null);
    assertTrue(cf.getColumn(ByteBufferUtil.bytes("name")).value().equals(hexToBytes("76616c")));
    qf = QueryFilter.getNamesFilter(Util.dk("rowExclude"), new QueryPath("Standard1", null, null), ByteBufferUtil.bytes("name"));
    cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
    assert cf == null;
}

Example 9 with SSTableWriter

use of org.apache.cassandra.io.sstable.SSTableWriter in project eiger by wlloyd.

the class SSTableExportTest method testExportSimpleCf.

@Test
public void testExportSimpleCf() throws IOException {
    File tempSS = tempSSTableFile("Keyspace1", "Standard1");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    //live for 42 seconds
    int nowInSec = (int) (System.currentTimeMillis() / 1000) + 42;
    // Add rowA
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
    cfamily.addColumn(null, new ExpiringColumn(ByteBufferUtil.bytes("colExp"), ByteBufferUtil.bytes("valExp"), System.currentTimeMillis(), 42, nowInSec));
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowB
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
    writer.append(Util.dk("rowB"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Standard1", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    JSONObject json = (JSONObject) JSONValue.parse(new FileReader(tempJson));
    JSONArray rowA = (JSONArray) json.get(asHex("rowA"));
    JSONArray colA = (JSONArray) rowA.get(0);
    assert hexToBytes((String) colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
    JSONArray colExp = (JSONArray) rowA.get(1);
    assert ((Long) colExp.get(4)) == 42;
    assert ((Long) colExp.get(5)) == nowInSec;
    JSONArray rowB = (JSONArray) json.get(asHex("rowB"));
    JSONArray colB = (JSONArray) rowB.get(0);
    assert colB.size() == 3;
    JSONArray rowExclude = (JSONArray) json.get(asHex("rowExclude"));
    assert rowExclude == null;
}

Example 10 with SSTableWriter

use of org.apache.cassandra.io.sstable.SSTableWriter in project eiger by wlloyd.

the class SSTableExportTest method testExportSuperCf.

@Test
public void testExportSuperCf() throws IOException {
    File tempSS = tempSSTableFile("Keyspace1", "Super4");
    ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4");
    SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
    // Add rowA
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superA"), ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
    writer.append(Util.dk("rowA"), cfamily);
    cfamily.clear();
    // Add rowB
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superB"), ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
    writer.append(Util.dk("rowB"), cfamily);
    cfamily.clear();
    // Add rowExclude
    cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superX"), ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
    writer.append(Util.dk("rowExclude"), cfamily);
    cfamily.clear();
    SSTableReader reader = writer.closeAndOpenReader();
    // Export to JSON and verify
    File tempJson = File.createTempFile("Super4", ".json");
    SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[] { asHex("rowExclude") });
    JSONObject json = (JSONObject) JSONValue.parse(new FileReader(tempJson));
    JSONObject rowA = (JSONObject) json.get(asHex("rowA"));
    JSONObject superA = (JSONObject) rowA.get(cfamily.getComparator().getString(ByteBufferUtil.bytes("superA")));
    JSONArray subColumns = (JSONArray) superA.get("subColumns");
    JSONArray colA = (JSONArray) subColumns.get(0);
    JSONObject rowExclude = (JSONObject) json.get(asHex("rowExclude"));
    assert hexToBytes((String) colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
    assert colA.size() == 3;
    assert rowExclude == null;
}