Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 41 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class UnalignedCheckpointsCancellationTest method test.

@Test
public void test() throws Exception {
    TestInvokable invokable = new TestInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(numChannels).setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    SingleCheckpointBarrierHandler unaligner = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    for (RuntimeEvent e : events) {
        if (e instanceof CancelCheckpointMarker) {
            unaligner.processCancellationBarrier((CancelCheckpointMarker) e, new InputChannelInfo(0, channel));
        } else if (e instanceof CheckpointBarrier) {
            unaligner.processBarrier((CheckpointBarrier) e, new InputChannelInfo(0, channel), false);
        } else {
            throw new IllegalArgumentException("unexpected event type: " + e);
        }
    }
    assertEquals("expectAbortCheckpoint", expectAbortCheckpoint, invokable.checkpointAborted);
    assertEquals("expectTriggerCheckpoint", expectTriggerCheckpoint, invokable.checkpointTriggered);
}

Example 42 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class UnalignedCheckpointsTest method testProcessCancellationBarrierBeforeProcessAndReceiveBarrier.

@Test
public void testProcessCancellationBarrierBeforeProcessAndReceiveBarrier() throws Exception {
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    handler.processCancellationBarrier(new CancelCheckpointMarker(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0));
    verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);
    // it would not trigger checkpoint since the respective cancellation barrier already
    // happened before
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);
}

Example 43 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class UnalignedCheckpointsTest method testProcessCancellationBarrierAfterProcessBarrier.

/**
 * Tests {@link
 * SingleCheckpointBarrierHandler#processCancellationBarrier(CancelCheckpointMarker,
 * InputChannelInfo)} abort the current pending checkpoint triggered by {@link
 * CheckpointBarrierHandler#processBarrier(CheckpointBarrier, InputChannelInfo)}.
 */
@Test
public void testProcessCancellationBarrierAfterProcessBarrier() throws Exception {
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    // should trigger respective checkpoint
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    assertTrue(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    testProcessCancellationBarrier(handler, invokable);
}

Example 44 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CheckpointBarrierTrackerTest method createCheckpointedInputGate.

// ------------------------------------------------------------------------
// Utils
// ------------------------------------------------------------------------
private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}

Example 45 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class StreamNetworkBenchmarkEnvironment method createInputGateWithMetrics.

private IndexedInputGate createInputGateWithMetrics(SingleInputGateFactory gateFactory, InputGateDeploymentDescriptor gateDescriptor, int gateIndex) {
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    final SingleInputGate singleGate = gateFactory.create(receiverEnv.createShuffleIOOwnerContext("receiving task[" + gateIndex + "]", taskMetricGroup.executionId(), taskMetricGroup), gateIndex, gateDescriptor, SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER);
    return new InputGateWithMetrics(singleGate, new SimpleCounter());
}