Examples with TrackedDataInputPlus org.apache.cassandra.io.util.TrackedDataInputPlus used on opensource projects

Example 1 with TrackedDataInputPlus

use of org.apache.cassandra.io.util.TrackedDataInputPlus in project cassandra by apache.

the class CassandraCompressedStreamReader method read.

/**
 * @return SSTable transferred
 * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
 */
@Override
// input needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
public SSTableMultiWriter read(DataInputPlus inputPlus) throws Throwable {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null) {
        // schema was dropped during streaming
        throw new IOException("CF " + tableId + " was dropped during streaming");
    }
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), pendingRepair, cfs.getTableName());
    StreamDeserializer deserializer = null;
    SSTableMultiWriter writer = null;
    try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance)) {
        TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
        deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
        writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
        String filename = writer.getFilename();
        int sectionIdx = 0;
        for (SSTableReader.PartitionPositionBounds section : sections) {
            assert cis.chunkBytesRead() <= totalSize;
            long sectionLength = section.upperPosition - section.lowerPosition;
            logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
            // skip to beginning of section inside chunk
            cis.position(section.lowerPosition);
            in.reset(0);
            while (in.getBytesRead() < sectionLength) {
                writePartition(deserializer, writer);
                // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
                session.progress(filename + '-' + fileSeqNum, ProgressInfo.Direction.IN, cis.chunkBytesRead(), totalSize);
            }
            assert in.getBytesRead() == sectionLength;
        }
        logger.info("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(cis.chunkBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName());
        if (writer != null)
            e = writer.abort(e);
        throw e;
    }
}

Example 2 with TrackedDataInputPlus

use of org.apache.cassandra.io.util.TrackedDataInputPlus in project cassandra by apache.

the class CassandraStreamReader method read.

/**
 * @param inputPlus where this reads data from
 * @return SSTable transferred
 * @throws IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
 */
// input needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
@Override
public SSTableMultiWriter read(DataInputPlus inputPlus) throws Throwable {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null)
        // schema was dropped during streaming
        throw new IllegalStateException("Table " + tableId + " was dropped during streaming");
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}', pendingRepair = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), cfs.getTableName(), pendingRepair);
    StreamDeserializer deserializer = null;
    SSTableMultiWriter writer = null;
    try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version)) {
        TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream);
        deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
        writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
        while (in.getBytesRead() < totalSize) {
            writePartition(deserializer, writer);
            // TODO move this to BytesReadTracker
            session.progress(writer.getFilename() + '-' + fileSeqNum, ProgressInfo.Direction.IN, in.getBytesRead(), totalSize);
        }
        logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(in.getBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName(), e);
        if (writer != null)
            e = writer.abort(e);
        throw e;
    }
}

Example 3 with TrackedDataInputPlus

use of org.apache.cassandra.io.util.TrackedDataInputPlus in project cassandra by apache.

the class BytesReadTrackerTest method internalTestSkipBytesAndReadFully.

public void internalTestSkipBytesAndReadFully(boolean inputStream) throws Exception {
    String testStr = "1234567890";
    byte[] testData = testStr.getBytes();
    DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        // read first 5 bytes
        byte[] out = new byte[5];
        reader.readFully(out, 0, 5);
        assertEquals("12345", new String(out));
        assertEquals(5, tracker.getBytesRead());
        // then skip 2 bytes
        reader.skipBytes(2);
        assertEquals(7, tracker.getBytesRead());
        // and read the rest
        out = new byte[3];
        reader.readFully(out);
        assertEquals("890", new String(out));
        assertEquals(10, tracker.getBytesRead());
        assertEquals(testData.length, tracker.getBytesRead());
    } finally {
        in.close();
    }
}

Example 4 with TrackedDataInputPlus

use of org.apache.cassandra.io.util.TrackedDataInputPlus in project cassandra by apache.

the class BytesReadTrackerTest method internalTestUnsignedRead.

public void internalTestUnsignedRead(boolean inputStream) throws Exception {
    byte[] testData;
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    try {
        // byte
        out.writeByte(0x1);
        // short
        out.writeShort(1);
        testData = baos.toByteArray();
    } finally {
        out.close();
    }
    DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        // byte = 1byte
        int b = reader.readUnsignedByte();
        assertEquals(b, 1);
        assertEquals(1, tracker.getBytesRead());
        // short = 2bytes
        int s = reader.readUnsignedShort();
        assertEquals(1, s);
        assertEquals(3, tracker.getBytesRead());
        assertEquals(testData.length, tracker.getBytesRead());
    } finally {
        in.close();
    }
}

Example 5 with TrackedDataInputPlus

use of org.apache.cassandra.io.util.TrackedDataInputPlus in project cassandra by apache.

the class DTestSerializer method deserialize.

public HintMessage deserialize(DataInputPlus in, int version) throws IOException {
    UUID hostId = UUIDSerializer.serializer.deserialize(in, version);
    long hintSize = in.readUnsignedVInt();
    TrackedDataInputPlus countingIn = new TrackedDataInputPlus(in);
    if (hintSize == 0)
        return new HintMessage(hostId, TableId.deserialize(countingIn));
    try {
        return new HintMessage(hostId, Hint.serializer.deserialize(countingIn, version));
    } catch (UnknownTableException e) {
        in.skipBytes(Ints.checkedCast(hintSize - countingIn.getBytesRead()));
        return new HintMessage(hostId, e.id);
    }
}