Search in sources :

Example 31 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class TestCheckpointedInputGateBuilder method buildMixedGate.

private SingleInputGate buildMixedGate(Integer... testChannelIds) throws IOException {
    Set<Integer> testChannelIdSet = new HashSet<>(Arrays.asList(testChannelIds));
    SingleInputGate gate = buildRemoteGate();
    InputChannel[] channels = new InputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        if (testChannelIdSet.contains(i)) {
            channels[i] = new TestInputChannel(gate, i, false, true);
        } else {
            channels[i] = gate.getChannel(i);
        }
    }
    gate.setInputChannels(channels);
    return gate;
}
Also used : TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) HashSet(java.util.HashSet)

Example 32 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink by splunk.

the class InputGateFairnessTest method testFairConsumptionLocalChannels.

@Test
public void testFairConsumptionLocalChannels() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
    try (BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42)) {
        // ----- create some source channels and fill them with one buffer each -----
        final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
        // ----- create reading side -----
        final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
        final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
        for (ResultPartition rp : resultPartitions) {
            rp.setup();
        }
        // seed one initial buffer
        sources[12].add(bufferConsumer.copy());
        setupInputGate(gate, inputChannels);
        // read all the buffers and the EOF event
        for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
            assertNotNull(gate.getNext());
            int min = Integer.MAX_VALUE;
            int max = 0;
            for (PipelinedSubpartition source : sources) {
                int size = source.getNumberOfQueuedBuffers();
                min = Math.min(min, size);
                max = Math.max(max, size);
            }
            assertTrue(max == min || max == min + 1);
            if (i % (2 * numberOfChannels) == 0) {
                // add three buffers to each channel, in random order
                fillRandom(sources, 3, bufferConsumer);
            }
        }
    // there is still more in the queues
    }
}
Also used : IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) TestBufferFactory(org.apache.flink.runtime.io.network.util.TestBufferFactory) SystemClock(org.apache.flink.util.clock.SystemClock) ThroughputCalculator(org.apache.flink.runtime.throughput.ThroughputCalculator) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) BufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferConsumer) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) InputChannelTestUtils.createDummyConnectionManager(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) Assert.fail(org.junit.Assert.fail) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) EventSerializer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) Assert.assertNotNull(org.junit.Assert.assertNotNull) BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) Collection(java.util.Collection) InputChannelBuilder(org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) BufferBuilderTestUtils.createFilledFinishedBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) NoOpBufferPool(org.apache.flink.runtime.io.network.buffer.NoOpBufferPool) UnpooledMemorySegmentProvider(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.UnpooledMemorySegmentProvider) Collections(java.util.Collections) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) SupplierWithException(org.apache.flink.util.function.SupplierWithException) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) BufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferConsumer) BufferBuilderTestUtils.createFilledFinishedBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) Test(org.junit.Test)

Example 33 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-mirror by flink-ci.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}
Also used : TestBufferFactory.createBuffer(org.apache.flink.runtime.io.network.util.TestBufferFactory.createBuffer) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) EventSerializer.toBuffer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer.toBuffer) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) SyncMailboxExecutor(org.apache.flink.runtime.mailbox.SyncMailboxExecutor) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)

Example 34 with InputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannel in project flink-ml by apache.

the class HeadOperator method endInput.

@Override
public void endInput() throws Exception {
    if (status == HeadOperatorStatus.RUNNING) {
        recordProcessor.processElement(new StreamRecord<>(IterationRecord.newEpochWatermark(0, "fake")));
    }
    // Since we choose to block here, we could not continue to process the barriers received
    // from the task inputs, which would block the precedent tasks from finishing since
    // they need to complete their final checkpoint. This is a temporary solution to this issue
    // that we will check the input channels, trigger all the checkpoints until we see
    // the EndOfPartitionEvent.
    checkState(getContainingTask().getEnvironment().getAllInputGates().length == 1);
    checkState(getContainingTask().getEnvironment().getAllInputGates()[0].getNumberOfInputChannels() == 1);
    InputChannel inputChannel = getContainingTask().getEnvironment().getAllInputGates()[0].getChannel(0);
    boolean endOfPartitionReceived = false;
    long lastTriggerCheckpointId = 0;
    while (!endOfPartitionReceived && status != HeadOperatorStatus.TERMINATED) {
        mailboxExecutor.tryYield();
        Thread.sleep(200);
        List<AbstractEvent> events = parseInputChannelEvents(inputChannel);
        for (AbstractEvent event : events) {
            if (event instanceof CheckpointBarrier) {
                CheckpointBarrier barrier = (CheckpointBarrier) event;
                if (barrier.getId() > lastTriggerCheckpointId) {
                    getContainingTask().triggerCheckpointAsync(new CheckpointMetaData(barrier.getId(), barrier.getTimestamp()), barrier.getCheckpointOptions());
                    lastTriggerCheckpointId = barrier.getId();
                }
            } else if (event instanceof EndOfPartitionEvent) {
                endOfPartitionReceived = true;
            }
        }
    }
    // By here we could step into the normal loop.
    while (status != HeadOperatorStatus.TERMINATED) {
        mailboxExecutor.yield();
    }
}
Also used : CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) InputChannel(org.apache.flink.runtime.io.network.partition.consumer.InputChannel) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) LocalInputChannel(org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel) AbstractEvent(org.apache.flink.runtime.event.AbstractEvent) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData)

Aggregations

InputChannel (org.apache.flink.runtime.io.network.partition.consumer.InputChannel)34 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)34 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)15 HashSet (java.util.HashSet)9 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)9 BufferPool (org.apache.flink.runtime.io.network.buffer.BufferPool)9 InputChannelTestUtils.createDummyConnectionManager (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager)9 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)9 EndOfPartitionEvent (org.apache.flink.runtime.io.network.api.EndOfPartitionEvent)7 IOException (java.io.IOException)6 ArrayList (java.util.ArrayList)6 Arrays (java.util.Arrays)6 Collection (java.util.Collection)6 Collections (java.util.Collections)6 Optional (java.util.Optional)6 IntStream (java.util.stream.IntStream)6 SubpartitionIndexRange (org.apache.flink.runtime.deployment.SubpartitionIndexRange)6 ConnectionManager (org.apache.flink.runtime.io.network.ConnectionManager)6 EventSerializer (org.apache.flink.runtime.io.network.api.serialization.EventSerializer)6 BufferBuilderTestUtils.createFilledFinishedBufferConsumer (org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer)6