Examples with KeyInputStream org.apache.hadoop.ozone.client.io.KeyInputStream used on opensource projects

Example 1 with KeyInputStream

use of org.apache.hadoop.ozone.client.io.KeyInputStream in project ozone by apache.

the class TestChunkStreams method testErrorReadGroupInputStream.

@Test
public void testErrorReadGroupInputStream() throws Exception {
    try (KeyInputStream groupInputStream = new KeyInputStream()) {
        String dataString = RandomStringUtils.randomAscii(500);
        byte[] buf = dataString.getBytes(UTF_8);
        int offset = 0;
        for (int i = 0; i < 5; i++) {
            int tempOffset = offset;
            BlockInputStream in = new BlockInputStream(null, 100, null, null, true, null) {

                private long pos = 0;

                private ByteArrayInputStream in = new ByteArrayInputStream(buf, tempOffset, 100);

                @Override
                public synchronized void seek(long pos) throws IOException {
                    throw new UnsupportedOperationException();
                }

                @Override
                public synchronized long getPos() {
                    return pos;
                }

                @Override
                public synchronized boolean seekToNewSource(long targetPos) throws IOException {
                    throw new UnsupportedOperationException();
                }

                @Override
                public synchronized int read() throws IOException {
                    return in.read();
                }

                @Override
                public synchronized int read(byte[] b, int off, int len) throws IOException {
                    int readLen = in.read(b, off, len);
                    pos += readLen;
                    return readLen;
                }
            };
            offset += 100;
            groupInputStream.addStream(in);
        }
        byte[] resBuf = new byte[600];
        // read 300 bytes first
        int len = groupInputStream.read(resBuf, 0, 340);
        assertEquals(3, groupInputStream.getCurrentStreamIndex());
        assertEquals(60, groupInputStream.getRemainingOfIndex(3));
        assertEquals(340, len);
        assertEquals(dataString.substring(0, 340), new String(resBuf, UTF_8).substring(0, 340));
        // read following 300 bytes, but only 200 left
        len = groupInputStream.read(resBuf, 340, 260);
        assertEquals(4, groupInputStream.getCurrentStreamIndex());
        assertEquals(0, groupInputStream.getRemainingOfIndex(4));
        assertEquals(160, len);
        assertEquals(dataString, new String(resBuf, UTF_8).substring(0, 500));
        // further read should get EOF
        len = groupInputStream.read(resBuf, 0, 1);
        // reached EOF, further read should get -1
        assertEquals(-1, len);
    }
}

Example 2 with KeyInputStream

use of org.apache.hadoop.ozone.client.io.KeyInputStream in project ozone by apache.

the class TestChunkInputStream method testChunkReadBuffers.

/**
 * Test to verify that data read from chunks is stored in a list of buffers
 * with max capacity equal to the bytes per checksum.
 */
public void testChunkReadBuffers() throws Exception {
    String keyName = getNewKeyName();
    int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE);
    byte[] inputData = writeRandomBytes(keyName, dataLength);
    KeyInputStream keyInputStream = getKeyInputStream(keyName);
    BlockInputStream block0Stream = keyInputStream.getBlockStreams().get(0);
    block0Stream.initialize();
    ChunkInputStream chunk0Stream = block0Stream.getChunkStreams().get(0);
    // To read 1 byte of chunk data, ChunkInputStream should get one full
    // checksum boundary worth of data from Container and store it in buffers.
    chunk0Stream.read(new byte[1]);
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
    // Read > checksum boundary of data from chunk0
    int readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2);
    byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen);
    validateData(inputData, 0, readData);
    // The first checksum boundary size of data was already existing in the
    // ChunkStream buffers. Once that data is read, the next checksum
    // boundary size of data will be fetched again to read the remaining data.
    // Hence there should be 1 checksum boundary size of data stored in the
    // ChunkStreams buffers at the end of the read.
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
    // Seek to a position in the third checksum boundary (so that current
    // buffers do not have the seeked position) and read > BYTES_PER_CHECKSUM
    // bytes of data. This should result in 2 * BYTES_PER_CHECKSUM amount of
    // data being read into the buffers. There should be 2 buffers in the
    // stream but the the first buffer should be released after it is read
    // and the second buffer should have BYTES_PER_CHECKSUM capacity.
    readDataLen = BYTES_PER_CHECKSUM + (BYTES_PER_CHECKSUM / 2);
    int offset = 2 * BYTES_PER_CHECKSUM + 1;
    readData = readDataFromChunk(chunk0Stream, offset, readDataLen);
    validateData(inputData, offset, readData);
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 2, 1, BYTES_PER_CHECKSUM);
    // Read the full chunk data - 1 and verify that all chunk data is read into
    // buffers. We read CHUNK_SIZE - 1 as otherwise all the buffers will be
    // released once all chunk data is read.
    readData = readDataFromChunk(chunk0Stream, 0, CHUNK_SIZE - 1);
    validateData(inputData, 0, readData);
    int expectedNumBuffers = CHUNK_SIZE / BYTES_PER_CHECKSUM;
    checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), expectedNumBuffers, expectedNumBuffers - 1, BYTES_PER_CHECKSUM);
    // Read the last byte of chunk and verify that the buffers are released.
    chunk0Stream.read(new byte[1]);
    Assert.assertNull("ChunkInputStream did not release buffers after " + "reaching EOF.", chunk0Stream.getCachedBuffers());
}

Example 3 with KeyInputStream

use of org.apache.hadoop.ozone.client.io.KeyInputStream in project ozone by apache.

the class TestChunkInputStream method testBufferRelease.

/**
 * Test that ChunkInputStream buffers are released as soon as the last byte
 * of the buffer is read.
 */
public void testBufferRelease() throws Exception {
    String keyName = getNewKeyName();
    int dataLength = CHUNK_SIZE;
    byte[] inputData = writeRandomBytes(keyName, dataLength);
    try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) {
        BlockInputStream block0Stream = keyInputStream.getBlockStreams().get(0);
        block0Stream.initialize();
        ChunkInputStream chunk0Stream = block0Stream.getChunkStreams().get(0);
        // Read checksum boundary - 1 bytes of data
        int readDataLen = BYTES_PER_CHECKSUM - 1;
        byte[] readData = readDataFromChunk(chunk0Stream, 0, readDataLen);
        validateData(inputData, 0, readData);
        // There should be 1 byte of data remaining in the buffer which is not
        // yet read. Hence, the buffer should not be released.
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        Assert.assertEquals(1, chunk0Stream.getCachedBuffers()[0].remaining());
        // Reading the last byte in the buffer should result in all the buffers
        // being released.
        readData = readDataFromChunk(chunk0Stream, 1);
        validateData(inputData, readDataLen, readData);
        Assert.assertNull("Chunk stream buffers not released after last byte is " + "read", chunk0Stream.getCachedBuffers());
        // Read more data to get the data till the next checksum boundary.
        readDataLen = BYTES_PER_CHECKSUM / 2;
        readData = readDataFromChunk(chunk0Stream, readDataLen);
        // There should be one buffer and the buffer should not be released as
        // there is data pending to be read from the buffer
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        ByteBuffer lastCachedBuffer = chunk0Stream.getCachedBuffers()[0];
        Assert.assertEquals(BYTES_PER_CHECKSUM - readDataLen, lastCachedBuffer.remaining());
        // Read more than the remaining data in buffer (but less than the next
        // checksum boundary).
        int position = (int) chunk0Stream.getPos();
        readDataLen = lastCachedBuffer.remaining() + BYTES_PER_CHECKSUM / 2;
        readData = readDataFromChunk(chunk0Stream, readDataLen);
        validateData(inputData, position, readData);
        // After reading the remaining data in the buffer, the buffer should be
        // released and next checksum size of data must be read into the buffers
        checkBufferSizeAndCapacity(chunk0Stream.getCachedBuffers(), 1, 0, BYTES_PER_CHECKSUM);
        // Verify that the previously cached buffer is released by comparing it
        // with the current cached buffer
        Assert.assertNotEquals(lastCachedBuffer, chunk0Stream.getCachedBuffers()[0]);
    }
}

Example 4 with KeyInputStream

use of org.apache.hadoop.ozone.client.io.KeyInputStream in project ozone by apache.

the class TestKeyInputStream method testReadAfterReplication.

private void testReadAfterReplication(boolean doUnbuffer) throws Exception {
    Assume.assumeTrue(getCluster().getHddsDatanodes().size() > 3);
    int dataLength = 2 * CHUNK_SIZE;
    String keyName = getNewKeyName();
    byte[] data = writeRandomBytes(keyName, dataLength);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(getVolumeName()).setBucketName(getBucketName()).setKeyName(keyName).setReplicationConfig(RatisReplicationConfig.getInstance(THREE)).build();
    OmKeyInfo keyInfo = getCluster().getOzoneManager().lookupKey(keyArgs);
    OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations();
    Assert.assertNotNull(locations);
    List<OmKeyLocationInfo> locationInfoList = locations.getLocationList();
    Assert.assertEquals(1, locationInfoList.size());
    OmKeyLocationInfo loc = locationInfoList.get(0);
    long containerID = loc.getContainerID();
    Assert.assertEquals(3, countReplicas(containerID, getCluster()));
    TestHelper.waitForContainerClose(getCluster(), containerID);
    List<DatanodeDetails> pipelineNodes = loc.getPipeline().getNodes();
    // read chunk data
    try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) {
        int b = keyInputStream.read();
        Assert.assertNotEquals(-1, b);
        if (doUnbuffer) {
            keyInputStream.unbuffer();
        }
        getCluster().shutdownHddsDatanode(pipelineNodes.get(0));
        // check that we can still read it
        assertReadFully(data, keyInputStream, dataLength - 1, 1);
    }
    // read chunk data with ByteBuffer
    try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) {
        int b = keyInputStream.read();
        Assert.assertNotEquals(-1, b);
        if (doUnbuffer) {
            keyInputStream.unbuffer();
        }
        getCluster().shutdownHddsDatanode(pipelineNodes.get(0));
        // check that we can still read it
        assertReadFullyUsingByteBuffer(data, keyInputStream, dataLength - 1, 1);
    }
}

Example 5 with KeyInputStream

use of org.apache.hadoop.ozone.client.io.KeyInputStream in project ozone by apache.

the class TestKeyInputStream method testReadChunkWithByteArray.

public void testReadChunkWithByteArray() throws Exception {
    String keyName = getNewKeyName();
    // write data spanning multiple blocks/chunks
    int dataLength = 2 * BLOCK_SIZE + (BLOCK_SIZE / 2);
    byte[] data = writeRandomBytes(keyName, dataLength);
    // read chunk data using Byte Array
    try (KeyInputStream keyInputStream = getKeyInputStream(keyName)) {
        int[] bufferSizeList = { BYTES_PER_CHECKSUM + 1, CHUNK_SIZE / 4, CHUNK_SIZE / 2, CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, BLOCK_SIZE - 1, BLOCK_SIZE, BLOCK_SIZE + 1, BLOCK_SIZE * 2 };
        for (int bufferSize : bufferSizeList) {
            assertReadFully(data, keyInputStream, bufferSize, 0);
            keyInputStream.seek(0);
        }
    }
}