Examples with BlockOutputStream org.apache.hadoop.hdds.scm.storage.BlockOutputStream used on opensource projects

Example 1 with BlockOutputStream

use of org.apache.hadoop.hdds.scm.storage.BlockOutputStream in project ozone by apache.

the class BlockOutputStreamEntry method writeOnRetry.

void writeOnRetry(long len) throws IOException {
    checkStream();
    BlockOutputStream out = (BlockOutputStream) getOutputStream();
    out.writeOnRetry(len);
    incCurrentPosition(len);
}

Example 2 with BlockOutputStream

use of org.apache.hadoop.hdds.scm.storage.BlockOutputStream in project ozone by apache.

the class TestBlockOutputStreamFlushDelay method testFlushChunk.

@Test
public void testFlushChunk() throws Exception {
    XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
    long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long totalOpCount = metrics.getTotalOpCount();
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = flushSize;
    // write data equal to 2 chunks
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    Assert.assertEquals(pendingWriteChunkCount + 2, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    // we have just written data equal flush Size = 2 chunks, at this time
    // buffer pool will have 2 buffers allocated worth of chunk size
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    // writtenDataLength as well flushedDataLength will be updated here
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
    Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
    // Now do a flush.
    key.flush();
    Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
    // The previously written data is equal to flushSize,so no action is
    // triggered when execute flush.
    Assert.assertEquals(pendingWriteChunkCount + 2, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    // Since the data in the buffer is already flushed, flush here will have
    // no impact on the counters and data structures
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    // No action is triggered when execute flush and BlockOutputStream will not
    // be updated.
    Assert.assertEquals(dataLength, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
    Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(totalOpCount + 4, metrics.getTotalOpCount());
    Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
    validateData(keyName, data1);
}

Example 3 with BlockOutputStream

use of org.apache.hadoop.hdds.scm.storage.BlockOutputStream in project ozone by apache.

the class TestBlockOutputStreamFlushDelay method testMultiChunkWrite.

@Test
public void testMultiChunkWrite() throws Exception {
    XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
    long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long totalOpCount = metrics.getTotalOpCount();
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = chunkSize + 50;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    Assert.assertEquals(pendingWriteChunkCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    // we have just written data equal flush Size > 1 chunk, at this time
    // buffer pool will have 2 buffers allocated worth of chunk size
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    // writtenDataLength as well flushedDataLength will be updated here
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    // since data written is still less than flushLength, flushLength will
    // still be 0.
    Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
    Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(putBlockCount + 1, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
    // flush ensures watchForCommit updates the total length acknowledged
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(totalOpCount + 4, metrics.getTotalOpCount());
    Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
    validateData(keyName, data1);
}

Example 4 with BlockOutputStream

use of org.apache.hadoop.hdds.scm.storage.BlockOutputStream in project ozone by apache.

the class TestBlockOutputStreamFlushDelay method testBufferCaching.

@Test
public void testBufferCaching() throws Exception {
    XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
    long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long totalOpCount = metrics.getTotalOpCount();
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = 50;
    int totalWriteDataLength = dataLength * 2;
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    // we have just written data less than a chunk size, the data will just sit
    // in the buffer, with only one buffer being allocated in the buffer pool
    Assert.assertEquals(1, blockOutputStream.getBufferPool().getSize());
    // Just the writtenDataLength will be updated here
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    // no data will be flushed till now
    Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
    Assert.assertEquals(pendingWriteChunkCount, XceiverClientManager.getXceiverClientMetrics().getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, XceiverClientManager.getXceiverClientMetrics().getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    // commitIndex2FlushedData Map will be empty here
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().isEmpty());
    // Total write data greater than or equal one chunk
    // size to make sure flush will sync data.
    key.write(data1);
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    // flush is a sync call, all pending operations will complete
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    // we have just written data less than a chunk size, the data will just sit
    // in the buffer, with only one buffer being allocated in the buffer pool
    Assert.assertEquals(1, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(0, blockOutputStream.getBufferPool().getBuffer(0).position());
    Assert.assertEquals(totalWriteDataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
    // flush ensures watchForCommit updates the total length acknowledged
    Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(writeChunkCount + 1, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(totalOpCount + 3, metrics.getTotalOpCount());
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
    validateData(keyName, data1);
}

Example 5 with BlockOutputStream

use of org.apache.hadoop.hdds.scm.storage.BlockOutputStream in project ozone by apache.

the class TestBlockOutputStreamFlushDelay method testFullBufferCondition.

@Test
public void testFullBufferCondition() throws Exception {
    XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
    long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
    long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
    long totalOpCount = metrics.getTotalOpCount();
    String keyName = getKeyName();
    OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
    int dataLength = maxFlushSize;
    // write data more than 1 chunk
    byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
    key.write(data1);
    // since its hitting the full bufferCondition, it will call watchForCommit
    // and completes atleast putBlock for first flushSize worth of data
    Assert.assertTrue(metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk) <= pendingWriteChunkCount + 2);
    Assert.assertTrue(metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock) <= pendingPutBlockCount + 1);
    Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
    KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
    Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
    OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
    Assert.assertTrue(stream instanceof BlockOutputStream);
    RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
    Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
    // writtenDataLength as well flushedDataLength will be updated here
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(maxFlushSize, blockOutputStream.getTotalDataFlushedLength());
    // since data equals to maxBufferSize is written, this will be a blocking
    // call and hence will wait for atleast flushSize worth of data to get
    // ack'd by all servers right here
    Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);
    // watchForCommit will clean up atleast one entry from the map where each
    // entry corresponds to flushSize worth of data
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 1);
    // Now do a flush. This will flush the data and update the flush length and
    // the map.
    key.flush();
    Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    // Since the data in the buffer is already flushed, flush here will have
    // no impact on the counters and data structures
    Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
    Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
    Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 1);
    // now close the stream, It will update the ack length after watchForCommit
    key.close();
    Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(writeChunkCount + 4, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
    Assert.assertEquals(putBlockCount + 3, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
    Assert.assertEquals(totalOpCount + 7, metrics.getTotalOpCount());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    // make sure the bufferPool is empty
    Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
    Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
    Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
    Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
    validateData(keyName, data1);
}