use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.
the class CreditBasedPartitionRequestClientHandlerTest method testReceiveBacklogAnnouncement.
/**
* Verifies that {@link NettyMessage.BacklogAnnouncement} can be handled correctly.
*/
@Test
public void testReceiveBacklogAnnouncement() throws Exception {
int bufferSize = 1024;
int numBuffers = 10;
NetworkBufferPool networkBufferPool = new NetworkBufferPool(numBuffers, bufferSize);
SingleInputGate inputGate = new SingleInputGateBuilder().setSegmentProvider(networkBufferPool).build();
RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
inputGate.setInputChannels(inputChannel);
try {
BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
inputGate.setBufferPool(bufferPool);
inputGate.setupChannels();
CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
handler.addInputChannel(inputChannel);
assertEquals(2, inputChannel.getNumberOfAvailableBuffers());
assertEquals(0, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
int backlog = 5;
NettyMessage.BacklogAnnouncement announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
handler.channelRead(null, announcement);
assertEquals(7, inputChannel.getNumberOfAvailableBuffers());
assertEquals(7, inputChannel.getNumberOfRequiredBuffers());
assertEquals(backlog, inputChannel.getSenderBacklog());
assertEquals(5, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
backlog = 12;
announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
handler.channelRead(null, announcement);
assertEquals(10, inputChannel.getNumberOfAvailableBuffers());
assertEquals(14, inputChannel.getNumberOfRequiredBuffers());
assertEquals(backlog, inputChannel.getSenderBacklog());
assertEquals(8, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
} finally {
releaseResource(inputGate, networkBufferPool);
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.
the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailable.
/**
* Verifies that {@link RemoteInputChannel} is enqueued in the pipeline for notifying credits,
* and verifies the behaviour of credit notification by triggering channel's writability
* changed.
*/
@Test
public void testNotifyCreditAvailable() throws Exception {
final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
final NetworkBufferAllocator allocator = new NetworkBufferAllocator(handler);
final EmbeddedChannel channel = new EmbeddedChannel(handler);
final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
inputChannels[0] = createRemoteInputChannel(inputGate, client);
inputChannels[1] = createRemoteInputChannel(inputGate, client);
try {
inputGate.setInputChannels(inputChannels);
final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
inputGate.setBufferPool(bufferPool);
inputGate.setupChannels();
inputChannels[0].requestSubpartition();
inputChannels[1].requestSubpartition();
// The two input channels should send partition requests
assertTrue(channel.isWritable());
Object readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
assertEquals(inputChannels[0].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
assertEquals(inputChannels[1].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
// The buffer response will take one available buffer from input channel, and it will
// trigger
// requesting (backlog + numExclusiveBuffers - numAvailableBuffers) floating buffers
final BufferResponse bufferResponse1 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[0].getInputChannelId(), 1, allocator);
final BufferResponse bufferResponse2 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[1].getInputChannelId(), 1, allocator);
handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse1);
handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse2);
assertEquals(2, inputChannels[0].getUnannouncedCredit());
assertEquals(2, inputChannels[1].getUnannouncedCredit());
channel.runPendingTasks();
// The two input channels should notify credits availability via the writable channel
readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(AddCredit.class));
assertEquals(inputChannels[0].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
assertEquals(2, ((AddCredit) readFromOutbound).credit);
readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(AddCredit.class));
assertEquals(inputChannels[1].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
assertEquals(2, ((AddCredit) readFromOutbound).credit);
assertNull(channel.readOutbound());
ByteBuf channelBlockingBuffer = blockChannel(channel);
// Trigger notify credits availability via buffer response on the condition of an
// un-writable channel
final BufferResponse bufferResponse3 = createBufferResponse(TestBufferFactory.createBuffer(32), 1, inputChannels[0].getInputChannelId(), 1, allocator);
handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse3);
assertEquals(1, inputChannels[0].getUnannouncedCredit());
assertEquals(0, inputChannels[1].getUnannouncedCredit());
channel.runPendingTasks();
// The input channel will not notify credits via un-writable channel
assertFalse(channel.isWritable());
assertNull(channel.readOutbound());
// Flush the buffer to make the channel writable again
channel.flush();
assertSame(channelBlockingBuffer, channel.readOutbound());
// The input channel should notify credits via channel's writability changed event
assertTrue(channel.isWritable());
readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(AddCredit.class));
assertEquals(1, ((AddCredit) readFromOutbound).credit);
assertEquals(0, inputChannels[0].getUnannouncedCredit());
assertEquals(0, inputChannels[1].getUnannouncedCredit());
// no more messages
assertNull(channel.readOutbound());
} finally {
releaseResource(inputGate, networkBufferPool);
channel.close();
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.
the class CreditBasedPartitionRequestClientHandlerTest method testReceiveBuffer.
/**
* Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a {@link
* BufferResponse} is received.
*/
@Test
public void testReceiveBuffer() throws Exception {
final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate);
try {
inputGate.setInputChannels(inputChannel);
final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
inputGate.setBufferPool(bufferPool);
inputGate.setupChannels();
final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
handler.addInputChannel(inputChannel);
final int backlog = 2;
final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog, new NetworkBufferAllocator(handler));
handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
assertEquals(2, inputChannel.getSenderBacklog());
} finally {
releaseResource(inputGate, networkBufferPool);
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.
the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailableAfterReleased.
/**
* Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
* message is not sent actually when this input channel is released.
*/
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
final EmbeddedChannel channel = new EmbeddedChannel(handler);
final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
try {
inputGate.setInputChannels(inputChannel);
final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
inputGate.setBufferPool(bufferPool);
inputGate.setupChannels();
inputChannel.requestSubpartition();
// This should send the partition request
Object readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
// Trigger request floating buffers via buffer response to notify credits available
final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
assertEquals(2, inputChannel.getUnannouncedCredit());
// Release the input channel
inputGate.close();
// it should send a close request after releasing the input channel,
// but will not notify credits for a released input channel.
readFromOutbound = channel.readOutbound();
assertThat(readFromOutbound, instanceOf(CloseRequest.class));
channel.runPendingTasks();
assertNull(channel.readOutbound());
} finally {
releaseResource(inputGate, networkBufferPool);
channel.close();
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.
the class TestCheckpointedInputGateBuilder method buildMixedGate.
private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
SingleInputGate gate = buildRemoteGate();
InputChannel[] channels = new InputChannel[numChannels];
for (int i = 0; i < numChannels; i++) {
if (testChannelIdSet.contains(i)) {
channels[i] = new TestInputChannel(gate, i, false, true);
} else {
channels[i] = gate.getChannel(i);
}
}
gate.setInputChannels(channels);
return gate;
}
Aggregations