Examples with BlockInputStream org.apache.hadoop.hdds.scm.storage.BlockInputStream used on opensource projects

Example 1 with BlockInputStream

use of org.apache.hadoop.hdds.scm.storage.BlockInputStream in project ozone by apache.

the class KeyInputStream method readWithStrategy.

synchronized int readWithStrategy(ByteReaderStrategy strategy) throws IOException {
    Preconditions.checkArgument(strategy != null);
    checkOpen();
    int buffLen = strategy.getTargetLength();
    int totalReadLen = 0;
    while (buffLen > 0) {
        // if we are at the last block and have read the entire block, return
        if (blockStreams.size() == 0 || (blockStreams.size() - 1 <= blockIndex && blockStreams.get(blockIndex).getRemaining() == 0)) {
            return totalReadLen == 0 ? EOF : totalReadLen;
        }
        // Get the current blockStream and read data from it
        BlockInputStream current = blockStreams.get(blockIndex);
        int numBytesToRead = Math.min(buffLen, (int) current.getRemaining());
        int numBytesRead = strategy.readFromBlock(current, numBytesToRead);
        if (numBytesRead != numBytesToRead) {
            // this case.
            throw new IOException(String.format("Inconsistent read for blockID=%s " + "length=%d numBytesToRead=%d numBytesRead=%d", current.getBlockID(), current.getLength(), numBytesToRead, numBytesRead));
        }
        totalReadLen += numBytesRead;
        buffLen -= numBytesRead;
        if (current.getRemaining() <= 0 && ((blockIndex + 1) < blockStreams.size())) {
            blockIndex += 1;
        }
    }
    return totalReadLen;
}

Example 2 with BlockInputStream

use of org.apache.hadoop.hdds.scm.storage.BlockInputStream in project ozone by apache.

the class TestChunkStreams method testErrorReadGroupInputStream.

@Test
public void testErrorReadGroupInputStream() throws Exception {
    try (KeyInputStream groupInputStream = new KeyInputStream()) {
        String dataString = RandomStringUtils.randomAscii(500);
        byte[] buf = dataString.getBytes(UTF_8);
        int offset = 0;
        for (int i = 0; i < 5; i++) {
            int tempOffset = offset;
            BlockInputStream in = new BlockInputStream(null, 100, null, null, true, null) {

                private long pos = 0;

                private ByteArrayInputStream in = new ByteArrayInputStream(buf, tempOffset, 100);

                @Override
                public synchronized void seek(long pos) throws IOException {
                    throw new UnsupportedOperationException();
                }

                @Override
                public synchronized long getPos() {
                    return pos;
                }

                @Override
                public synchronized boolean seekToNewSource(long targetPos) throws IOException {
                    throw new UnsupportedOperationException();
                }

                @Override
                public synchronized int read() throws IOException {
                    return in.read();
                }

                @Override
                public synchronized int read(byte[] b, int off, int len) throws IOException {
                    int readLen = in.read(b, off, len);
                    pos += readLen;
                    return readLen;
                }
            };
            offset += 100;
            groupInputStream.addStream(in);
        }
        byte[] resBuf = new byte[600];
        // read 300 bytes first
        int len = groupInputStream.read(resBuf, 0, 340);
        assertEquals(3, groupInputStream.getCurrentStreamIndex());
        assertEquals(60, groupInputStream.getRemainingOfIndex(3));
        assertEquals(340, len);
        assertEquals(dataString.substring(0, 340), new String(resBuf, UTF_8).substring(0, 340));
        // read following 300 bytes, but only 200 left
        len = groupInputStream.read(resBuf, 340, 260);
        assertEquals(4, groupInputStream.getCurrentStreamIndex());
        assertEquals(0, groupInputStream.getRemainingOfIndex(4));
        assertEquals(160, len);
        assertEquals(dataString, new String(resBuf, UTF_8).substring(0, 500));
        // further read should get EOF
        len = groupInputStream.read(resBuf, 0, 1);
        // reached EOF, further read should get -1
        assertEquals(-1, len);
    }
}

Example 3 with BlockInputStream

use of org.apache.hadoop.hdds.scm.storage.BlockInputStream in project ozone by apache.

the class TestChunkInputStream method testChunkReadBuffers.

/**
 * Test to verify that data read from chunks is stored in a list of buffers
 * with max capacity equal to the bytes per checksum.
 */
public void testChunkReadBuffers() throws Exception {
    String keyName = getNewKeyName();
    int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE);
    byte[] inputData = writeRandomBytes(keyName, dataLength);
    KeyInputStream keyInputStream = getKeyInputStream(keyName);
    BlockInputStream block0Stream = keyInputStream.getBlockStreams().get(0);
    block0Stream.initialize();
    ChunkInputStream chunk0Stream = block0Stream.getChunkStreams().get(0);
    // To read 1 byte of chunk data, ChunkInputStream should get one full
    // checksum boundary worth of data from Container and store it in buffers.
    chunk0Stream.read(new byte[1]);
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
    // Read > checksum boundary of data from chunk0
    int readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2);
    byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen);
    validateData(inputData, 0, readData);
    // The first checksum boundary size of data was already existing in the
    // ChunkStream buffers. Once that data is read, the next checksum
    // boundary size of data will be fetched again to read the remaining data.
    // Hence there should be 1 checksum boundary size of data stored in the
    // ChunkStreams buffers at the end of the read.
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
    // Seek to a position in the third checksum boundary (so that current
    // buffers do not have the seeked position) and read > BYTES_PER_CHECKSUM
    // bytes of data. This should result in 2 * BYTES_PER_CHECKSUM amount of
    // data being read into the buffers. There should be 2 buffers in the
    // stream but the the first buffer should be released after it is read
    // and the second buffer should have BYTES_PER_CHECKSUM capacity.
    readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2);
    int offset = 2 * BYTES_PER_CHECKSUM + 1;
    readData = readDataFromChunk(chunk0Stream, offset, readDataLen);
    validateData(inputData, offset, readData);
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 2, 1, BYTES_PER_CHECKSUM);
    // Read the full chunk data - 1 and verify that all chunk data is read into
    // buffers. We read CHUNK_SIZE - 1 as otherwise all the buffers will be
    // released once all chunk data is read.
    readData = readDataFromChunk(chunk0Stream, 0, CHUNK_SIZE - 1);
    validateData(inputData, 0, readData);
    int expectedNumBuffers = CHUNK_SIZE / BYTES_PER_CHECKSUM;
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), expectedNumBuffers, expectedNumBuffers - 1, BYTES_PER_CHECKSUM);
    // Read the last byte of chunk and verify that the buffers are released.
    chunk0Stream.read(new byte[1]);
    Assert.assertNull("ChunkInputStream did not release buffers after " + "reaching EOF.", chunk0Stream.getCachedBuffers());
}

Example 4 with BlockInputStream

use of org.apache.hadoop.hdds.scm.storage.BlockInputStream in project ozone by apache.

the class TestChunkInputStream method testBufferRelease.

/**
 * Test that ChunkInputStream buffers are released as soon as the last byte
 * of the buffer is read.
 */
public void testBufferRelease() throws Exception {
    String keyName = getNewKeyName();
    int dataLength = CHUNK_SIZE;
    byte[] inputData = writeRandomBytes(keyName, dataLength);
    try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) {
        BlockInputStream block0Stream = keyInputStream.getBlockStreams().get(0);
        block0Stream.initialize();
        ChunkInputStream chunk0Stream = block0Stream.getChunkStreams().get(0);
        // Read checksum boundary - 1 bytes of data
        int readDataLen = BYTES_PER_CHECKSUM - 1;
        byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen);
        validateData(inputData, 0, readData);
        // There should be 1 byte of data remaining in the buffer which is not
        // yet read. Hence, the buffer should not be released.
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        Assert.assertEquals(1, chunk0Stream.getCachedBuffers()[0].remaining());
        // Reading the last byte in the buffer should result in all the buffers
        // being released.
        readData = readDataFromChunk(chunk0Stream, 1);
        validateData(inputData, readDataLen, readData);
        Assert.assertNull("Chunk stream buffers not released after last byte is " + "read", chunk0Stream.getCachedBuffers());
        // Read more data to get the data till the next checksum boundary.
        readDataLen = BYTES_PER_CHECKSUM / 2;
        readData = readDataFromChunk(chunk0Stream, readDataLen);
        // There should be one buffer and the buffer should not be released as
        // there is data pending to be read from the buffer
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        ByteBuffer lastCachedBuffer = chunk0Stream.getCachedBuffers()[0];
        Assert.assertEquals(BYTES_PER_CHECKSUM - readDataLen, lastCachedBuffer.remaining());
        // Read more than the remaining data in buffer (but less than the next
        // checksum boundary).
        int position = (int) chunk0Stream.getPos();
        readDataLen = lastCachedBuffer.remaining() + BYTES_PER_CHECKSUM / 2;
        readData = readDataFromChunk(chunk0Stream, readDataLen);
        validateData(inputData, position, readData);
        // After reading the remaining data in the buffer, the buffer should be
        // released and next checksum size of data must be read into the buffers
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        // Verify that the previously cached buffer is released by comparing it
        // with the current cached buffer
        Assert.assertNotEquals(lastCachedBuffer, chunk0Stream.getCachedBuffers()[0]);
    }
}

Example 5 with BlockInputStream

use of org.apache.hadoop.hdds.scm.storage.BlockInputStream in project ozone by apache.

the class TestKeyInputStream method testInputStreams.

public void testInputStreams() throws Exception {
    String keyName = getNewKeyName();
    int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE) + 1;
    writeRandomBytes(keyName, dataLength);
    KeyInputStream keyInputStream = getKeyInputStream(keyName);
    // Verify BlockStreams and ChunkStreams
    int expectedNumBlockStreams = BufferUtils.getNumberOfBins(dataLength, BLOCK_SIZE);
    List<BlockInputStream> blockStreams = keyInputStream.getBlockStreams();
    Assert.assertEquals(expectedNumBlockStreams, blockStreams.size());
    int readBlockLength = 0;
    for (BlockInputStream blockStream : blockStreams) {
        int blockStreamLength = Math.min(BLOCK_SIZE, dataLength - readBlockLength);
        Assert.assertEquals(blockStreamLength, blockStream.getLength());
        int expectedNumChunkStreams = BufferUtils.getNumberOfBins(blockStreamLength, CHUNK_SIZE);
        blockStream.initialize();
        List<ChunkInputStream> chunkStreams = blockStream.getChunkStreams();
        Assert.assertEquals(expectedNumChunkStreams, chunkStreams.size());
        int readChunkLength = 0;
        for (ChunkInputStream chunkStream : chunkStreams) {
            int chunkStreamLength = Math.min(CHUNK_SIZE, blockStreamLength - readChunkLength);
            Assert.assertEquals(chunkStreamLength, chunkStream.getRemaining());
            readChunkLength += chunkStreamLength;
        }
        readBlockLength += blockStreamLength;
    }
}