use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.
the class TestCheckpointedInputGateBuilder method buildMixedGate.
private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
SingleInputGate gate = buildRemoteGate();
InputChannel[] channels = new InputChannel[numChannels];
for (int i = 0; i < numChannels; i++) {
if (testChannelIdSet.contains(i)) {
channels[i] = new TestInputChannel(gate, i, false, true);
} else {
channels[i] = gate.getChannel(i);
}
}
gate.setInputChannels(channels);
return gate;
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.
the class InputGateFairnessTest method testFairConsumptionLocalChannels.
@Test
public void testFairConsumptionLocalChannels() throws Exception {
final int numberOfChannels = 37;
final int buffersPerChannel = 27;
PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
try (BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42)) {
// ----- create some source channels and fill them with one buffer each -----
final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
// ----- create reading side -----
final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
for (ResultPartition rp : resultPartitions) {
rp.setup();
}
// seed one initial buffer
sources[12].add(bufferConsumer.copy());
setupInputGate(gate, inputChannels);
// read all the buffers and the EOF event
for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
assertNotNull(gate.getNext());
int min = Integer.MAX_VALUE;
int max = 0;
for (PipelinedSubpartition source : sources) {
int size = source.getNumberOfQueuedBuffers();
min = Math.min(min, size);
max = Math.max(max, size);
}
assertTrue(max == min || max == min + 1);
if (i % (2 * numberOfChannels) == 0) {
// add three buffers to each channel, in random order
fillRandom(sources, 3, bufferConsumer);
}
}
// there is still more in the queues
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.
the class AlternatingCheckpointsTest method testBarrierHandling.
private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
final long barrierId = 123L;
ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
gate.setInputChannels(fast, slow);
SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
if (checkpointType.isSavepoint()) {
fast.setBlocked(true);
slow.setBlocked(true);
}
CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
Buffer barrier = barrier(barrierId, 1, options);
send(barrier.retainBuffer(), fast, checkpointedGate);
assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
send(barrier.retainBuffer(), slow, checkpointedGate);
assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
if (checkpointType.isSavepoint()) {
for (InputChannel channel : gate.getInputChannels().values()) {
assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
}
}
}
use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-ml by apache.
the class HeadOperator method endInput.
@Override
public void endInput() throws Exception {
if (status == HeadOperatorStatus.RUNNING) {
recordProcessor.processElement(new StreamRecord<>(IterationRecord.newEpochWatermark(0, "fake")));
}
// Since we choose to block here, we could not continue to process the barriers received
// from the task inputs, which would block the precedent tasks from finishing since
// they need to complete their final checkpoint. This is a temporary solution to this issue
// that we will check the input channels, trigger all the checkpoints until we see
// the EndOfPartitionEvent.
checkState(getContainingTask().getEnvironment().getAllInputGates().length == 1);
checkState(getContainingTask().getEnvironment().getAllInputGates()[0].getNumberOfInputChannels() == 1);
InputChannel inputChannel = getContainingTask().getEnvironment().getAllInputGates()[0].getChannel(0);
boolean endOfPartitionReceived = false;
long lastTriggerCheckpointId = 0;
while (!endOfPartitionReceived && status != HeadOperatorStatus.TERMINATED) {
mailboxExecutor.tryYield();
Thread.sleep(200);
List<AbstractEvent> events = parseInputChannelEvents(inputChannel);
for (AbstractEvent event : events) {
if (event instanceof CheckpointBarrier) {
CheckpointBarrier barrier = (CheckpointBarrier) event;
if (barrier.getId() > lastTriggerCheckpointId) {
getContainingTask().triggerCheckpointAsync(new CheckpointMetaData(barrier.getId(), barrier.getTimestamp()), barrier.getCheckpointOptions());
lastTriggerCheckpointId = barrier.getId();
}
} else if (event instanceof EndOfPartitionEvent) {
endOfPartitionReceived = true;
}
}
}
// By here we could step into the normal loop.
while (status != HeadOperatorStatus.TERMINATED) {
mailboxExecutor.yield();
}
}
Aggregations