Examples with TrackedInputStream org.apache.cassandra.io.util.TrackedInputStream used on opensource projects

Example 1 with TrackedInputStream

use of org.apache.cassandra.io.util.TrackedInputStream in project cassandra by apache.

the class CompressedStreamReader method read.

/**
     * @return SSTable transferred
     * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
     */
@Override
// channel needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null) {
        // schema was dropped during streaming
        throw new IOException("CF " + tableId + " was dropped during streaming");
    }
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), session.getPendingRepair(), cfs.getTableName());
    CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance);
    TrackedInputStream in = new TrackedInputStream(cis);
    StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
    SSTableMultiWriter writer = null;
    try {
        writer = createWriter(cfs, totalSize, repairedAt, session.getPendingRepair(), format);
        String filename = writer.getFilename();
        int sectionIdx = 0;
        for (Pair<Long, Long> section : sections) {
            assert cis.getTotalCompressedBytesRead() <= totalSize;
            long sectionLength = section.right - section.left;
            logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
            // skip to beginning of section inside chunk
            cis.position(section.left);
            in.reset(0);
            while (in.getBytesRead() < sectionLength) {
                writePartition(deserializer, writer);
                // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
                session.progress(filename, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize);
            }
        }
        logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(cis.getTotalCompressedBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName());
        if (writer != null) {
            writer.abort(e);
        }
        if (extractIOExceptionCause(e).isPresent())
            throw e;
        throw Throwables.propagate(e);
    }
}

Example 2 with TrackedInputStream

use of org.apache.cassandra.io.util.TrackedInputStream in project cassandra by apache.

the class BytesReadTrackerTest method internalTestSkipBytesAndReadFully.

public void internalTestSkipBytesAndReadFully(boolean inputStream) throws Exception {
    String testStr = "1234567890";
    byte[] testData = testStr.getBytes();
    DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        // read first 5 bytes
        byte[] out = new byte[5];
        reader.readFully(out, 0, 5);
        assertEquals("12345", new String(out));
        assertEquals(5, tracker.getBytesRead());
        // then skip 2 bytes
        reader.skipBytes(2);
        assertEquals(7, tracker.getBytesRead());
        // and read the rest
        out = new byte[3];
        reader.readFully(out);
        assertEquals("890", new String(out));
        assertEquals(10, tracker.getBytesRead());
        assertEquals(testData.length, tracker.getBytesRead());
    } finally {
        in.close();
    }
}

Example 3 with TrackedInputStream

use of org.apache.cassandra.io.util.TrackedInputStream in project cassandra by apache.

the class BytesReadTrackerTest method internalTestUnsignedRead.

public void internalTestUnsignedRead(boolean inputStream) throws Exception {
    byte[] testData;
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    try {
        // byte
        out.writeByte(0x1);
        // short
        out.writeShort(1);
        testData = baos.toByteArray();
    } finally {
        out.close();
    }
    DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        // byte = 1byte
        int b = reader.readUnsignedByte();
        assertEquals(b, 1);
        assertEquals(1, tracker.getBytesRead());
        // short = 2bytes
        int s = reader.readUnsignedShort();
        assertEquals(1, s);
        assertEquals(3, tracker.getBytesRead());
        assertEquals(testData.length, tracker.getBytesRead());
    } finally {
        in.close();
    }
}

Example 4 with TrackedInputStream

use of org.apache.cassandra.io.util.TrackedInputStream in project cassandra by apache.

the class BytesReadTrackerTest method internalTestReadLine.

public void internalTestReadLine(boolean inputStream) throws Exception {
    DataInputStream in = new DataInputStream(new ByteArrayInputStream("1".getBytes()));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        String line = reader.readLine();
        if (inputStream)
            assertEquals(line, "1");
        else
            fail("Should have thrown UnsupportedOperationException");
    } catch (UnsupportedOperationException e) {
        if (inputStream)
            fail("Should have not thrown UnsupportedOperationException");
    } finally {
        in.close();
    }
}

Example 5 with TrackedInputStream

use of org.apache.cassandra.io.util.TrackedInputStream in project cassandra by apache.

the class BytesReadTrackerTest method internalTestBytesRead.

public void internalTestBytesRead(boolean inputStream) throws Exception {
    byte[] testData;
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    try {
        // boolean
        out.writeBoolean(true);
        // byte
        out.writeByte(0x1);
        // char
        out.writeChar('a');
        // short
        out.writeShort(1);
        // int
        out.writeInt(1);
        // long
        out.writeLong(1L);
        // float
        out.writeFloat(1.0f);
        // double
        out.writeDouble(1.0d);
        // String
        out.writeUTF("abc");
        testData = baos.toByteArray();
    } finally {
        out.close();
    }
    DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData));
    BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in);
    DataInputPlus reader = inputStream ? new DataInputPlus.DataInputStreamPlus((TrackedInputStream) tracker) : (DataInputPlus) tracker;
    try {
        // boolean = 1byte
        boolean bool = reader.readBoolean();
        assertTrue(bool);
        assertEquals(1, tracker.getBytesRead());
        // byte = 1byte
        byte b = reader.readByte();
        assertEquals(b, 0x1);
        assertEquals(2, tracker.getBytesRead());
        // char = 2byte
        char c = reader.readChar();
        assertEquals('a', c);
        assertEquals(4, tracker.getBytesRead());
        // short = 2bytes
        short s = reader.readShort();
        assertEquals(1, s);
        assertEquals((short) 6, tracker.getBytesRead());
        // int = 4bytes
        int i = reader.readInt();
        assertEquals(1, i);
        assertEquals(10, tracker.getBytesRead());
        // long = 8bytes
        long l = reader.readLong();
        assertEquals(1L, l);
        assertEquals(18, tracker.getBytesRead());
        // float = 4bytes
        float f = reader.readFloat();
        assertEquals(1.0f, f, 0);
        assertEquals(22, tracker.getBytesRead());
        // double = 8bytes
        double d = reader.readDouble();
        assertEquals(1.0d, d, 0);
        assertEquals(30, tracker.getBytesRead());
        // String("abc") = 2(string size) + 3 = 5 bytes
        String str = reader.readUTF();
        assertEquals("abc", str);
        assertEquals(35, tracker.getBytesRead());
        assertEquals(testData.length, tracker.getBytesRead());
    } finally {
        in.close();
    }
    tracker.reset(0);
    assertEquals(0, tracker.getBytesRead());
}