use of org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream in project ozone by apache.
the class TestBlockOutputStreamFlushDelay method testFlushChunk.
@Test
public void testFlushChunk() throws Exception {
XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
int dataLength = flushSize;
// write data equal to 2 chunks
byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
key.write(data1);
Assert.assertEquals(pendingWriteChunkCount + 2, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
Assert.assertTrue(stream instanceof BlockOutputStream);
RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
// we have just written data equal flush Size = 2 chunks, at this time
// buffer pool will have 2 buffers allocated worth of chunk size
Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
// writtenDataLength as well flushedDataLength will be updated here
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
// Now do a flush.
key.flush();
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
// The previously written data is equal to flushSize,so no action is
// triggered when execute flush.
Assert.assertEquals(pendingWriteChunkCount + 2, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
// No action is triggered when execute flush and BlockOutputStream will not
// be updated.
Assert.assertEquals(dataLength, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
// now close the stream, It will update the ack length after watchForCommit
key.close();
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
// make sure the bufferPool is empty
Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(totalOpCount + 4, metrics.getTotalOpCount());
Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
validateData(keyName, data1);
}
use of org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream in project ozone by apache.
the class TestBlockOutputStreamFlushDelay method testMultiChunkWrite.
@Test
public void testMultiChunkWrite() throws Exception {
XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
int dataLength = chunkSize + 50;
// write data more than 1 chunk
byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
key.write(data1);
Assert.assertEquals(pendingWriteChunkCount + 1, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
Assert.assertTrue(stream instanceof BlockOutputStream);
RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
// we have just written data equal flush Size > 1 chunk, at this time
// buffer pool will have 2 buffers allocated worth of chunk size
Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
// writtenDataLength as well flushedDataLength will be updated here
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
// since data written is still less than flushLength, flushLength will
// still be 0.
Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
// Now do a flush. This will flush the data and update the flush length and
// the map.
key.flush();
Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 1, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
// flush ensures watchForCommit updates the total length acknowledged
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
// now close the stream, It will update the ack length after watchForCommit
key.close();
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
// make sure the bufferPool is empty
Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(totalOpCount + 4, metrics.getTotalOpCount());
Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
validateData(keyName, data1);
}
use of org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream in project ozone by apache.
the class TestBlockOutputStreamFlushDelay method testBufferCaching.
@Test
public void testBufferCaching() throws Exception {
XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
int dataLength = 50;
int totalWriteDataLength = dataLength * 2;
byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
key.write(data1);
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
Assert.assertTrue(stream instanceof BlockOutputStream);
RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
// we have just written data less than a chunk size, the data will just sit
// in the buffer, with only one buffer being allocated in the buffer pool
Assert.assertEquals(1, blockOutputStream.getBufferPool().getSize());
// Just the writtenDataLength will be updated here
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
// no data will be flushed till now
Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(pendingWriteChunkCount, XceiverClientManager.getXceiverClientMetrics().getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, XceiverClientManager.getXceiverClientMetrics().getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// commitIndex2FlushedData Map will be empty here
Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().isEmpty());
// Total write data greater than or equal one chunk
// size to make sure flush will sync data.
key.write(data1);
// Now do a flush. This will flush the data and update the flush length and
// the map.
key.flush();
// flush is a sync call, all pending operations will complete
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// we have just written data less than a chunk size, the data will just sit
// in the buffer, with only one buffer being allocated in the buffer pool
Assert.assertEquals(1, blockOutputStream.getBufferPool().getSize());
Assert.assertEquals(0, blockOutputStream.getBufferPool().getBuffer(0).position());
Assert.assertEquals(totalWriteDataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalDataFlushedLength());
Assert.assertEquals(0, blockOutputStream.getCommitIndex2flushedDataMap().size());
// flush ensures watchForCommit updates the total length acknowledged
Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
// now close the stream, It will update the ack length after watchForCommit
key.close();
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 1, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(totalOpCount + 3, metrics.getTotalOpCount());
// make sure the bufferPool is empty
Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(totalWriteDataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
validateData(keyName, data1);
}
use of org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream in project ozone by apache.
the class TestBlockOutputStreamFlushDelay method testFullBufferCondition.
@Test
public void testFullBufferCondition() throws Exception {
XceiverClientMetrics metrics = XceiverClientManager.getXceiverClientMetrics();
long writeChunkCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk);
long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0);
int dataLength = maxFlushSize;
// write data more than 1 chunk
byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
key.write(data1);
// since its hitting the full bufferCondition, it will call watchForCommit
// and completes atleast putBlock for first flushSize worth of data
Assert.assertTrue(metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk) <= pendingWriteChunkCount + 2);
Assert.assertTrue(metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock) <= pendingPutBlockCount + 1);
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
Assert.assertTrue(stream instanceof BlockOutputStream);
RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
// writtenDataLength as well flushedDataLength will be updated here
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(maxFlushSize, blockOutputStream.getTotalDataFlushedLength());
// since data equals to maxBufferSize is written, this will be a blocking
// call and hence will wait for atleast flushSize worth of data to get
// ack'd by all servers right here
Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);
// watchForCommit will clean up atleast one entry from the map where each
// entry corresponds to flushSize worth of data
Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 1);
// Now do a flush. This will flush the data and update the flush length and
// the map.
key.flush();
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 1);
// now close the stream, It will update the ack length after watchForCommit
key.close();
Assert.assertEquals(pendingWriteChunkCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount, metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 4, metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 3, metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(totalOpCount + 7, metrics.getTotalOpCount());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
// make sure the bufferPool is empty
Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
validateData(keyName, data1);
}
use of org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream in project ozone by apache.
the class TestBlockOutputStreamWithFailuresFlushDelay method testWatchForCommitWithSingleNodeRatis.
private void testWatchForCommitWithSingleNodeRatis() throws Exception {
String keyName = getKeyName();
OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0, ReplicationFactor.ONE);
int dataLength = maxFlushSize + chunkSize;
// write data more than 1 chunk
byte[] data1 = ContainerTestHelper.getFixedLengthString(keyString, dataLength).getBytes(UTF_8);
key.write(data1);
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();
Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
OutputStream stream = keyOutputStream.getStreamEntries().get(0).getOutputStream();
Assert.assertTrue(stream instanceof BlockOutputStream);
RatisBlockOutputStream blockOutputStream = (RatisBlockOutputStream) stream;
// we have just written data more than flush Size(2 chunks), at this time
// buffer pool will have 4 buffers allocated worth of chunk size
Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
// writtenDataLength as well flushedDataLength will be updated here
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(maxFlushSize, blockOutputStream.getTotalDataFlushedLength());
// since data equals to maxBufferSize is written, this will be a blocking
// call and hence will wait for atleast flushSize worth of data to get
// ack'd by all servers right here
Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);
// watchForCommit will clean up atleast one entry from the map where each
// entry corresponds to flushSize worth of data
Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 1);
// Now do a flush. This will flush the data and update the flush length and
// the map.
key.flush();
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
Assert.assertEquals(dataLength, blockOutputStream.getTotalDataFlushedLength());
// flush will make sure one more entry gets updated in the map
Assert.assertTrue(blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);
XceiverClientRatis raftClient = (XceiverClientRatis) blockOutputStream.getXceiverClient();
Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
// Close the containers on the Datanode and write more data
TestHelper.waitForContainerClose(key, cluster);
// 4 writeChunks = maxFlushSize + 2 putBlocks will be discarded here
// once exception is hit
key.write(data1);
// As a part of handling the exception, 4 failed writeChunks will be
// rewritten plus one partial chunk plus two putBlocks for flushSize
// and one flush for partial chunk
key.flush();
Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream.getIoException()) instanceof ContainerNotOpenException);
// Make sure the retryCount is reset after the exception is handled
Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
// commitInfoMap will remain intact as there is no server failure
Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
Assert.assertEquals(2, keyOutputStream.getStreamEntries().size());
// now close the stream, It will update the ack length after watchForCommit
key.close();
// make sure the bufferPool is empty
Assert.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
// Written the same data twice
String dataString = new String(data1, UTF_8);
validateData(keyName, dataString.concat(dataString).getBytes(UTF_8));
}
Aggregations