Examples with SSTableMultiWriter org.apache.cassandra.io.sstable.SSTableMultiWriter used on opensource projects

Example 1 with SSTableMultiWriter

use of org.apache.cassandra.io.sstable.SSTableMultiWriter in project cassandra by apache.

the class CompressedStreamReader method read.

/**
     * @return SSTable transferred
     * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
     */
@Override
// channel needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null) {
        // schema was dropped during streaming
        throw new IOException("CF " + tableId + " was dropped during streaming");
    }
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), session.getPendingRepair(), cfs.getTableName());
    CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance);
    TrackedInputStream in = new TrackedInputStream(cis);
    StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
    SSTableMultiWriter writer = null;
    try {
        writer = createWriter(cfs, totalSize, repairedAt, session.getPendingRepair(), format);
        String filename = writer.getFilename();
        int sectionIdx = 0;
        for (Pair<Long, Long> section : sections) {
            assert cis.getTotalCompressedBytesRead() <= totalSize;
            long sectionLength = section.right - section.left;
            logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
            // skip to beginning of section inside chunk
            cis.position(section.left);
            in.reset(0);
            while (in.getBytesRead() < sectionLength) {
                writePartition(deserializer, writer);
                // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
                session.progress(filename, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize);
            }
        }
        logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(cis.getTotalCompressedBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName());
        if (writer != null) {
            writer.abort(e);
        }
        if (extractIOExceptionCause(e).isPresent())
            throw e;
        throw Throwables.propagate(e);
    }
}

Example 2 with SSTableMultiWriter

use of org.apache.cassandra.io.sstable.SSTableMultiWriter in project cassandra by apache.

the class StreamReader method read.

/**
     * @param channel where this reads data from
     * @return SSTable transferred
     * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
     */
// channel needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null) {
        // schema was dropped during streaming
        throw new IOException("CF " + tableId + " was dropped during streaming");
    }
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}', pendingRepair = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), cfs.getTableName(), session.getPendingRepair());
    TrackedInputStream in = new TrackedInputStream(new LZFInputStream(Channels.newInputStream(channel)));
    StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
    SSTableMultiWriter writer = null;
    try {
        writer = createWriter(cfs, totalSize, repairedAt, session.getPendingRepair(), format);
        while (in.getBytesRead() < totalSize) {
            writePartition(deserializer, writer);
            // TODO move this to BytesReadTracker
            session.progress(writer.getFilename(), ProgressInfo.Direction.IN, in.getBytesRead(), totalSize);
        }
        logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(in.getBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName(), e);
        if (writer != null) {
            writer.abort(e);
        }
        throw Throwables.propagate(e);
    }
}