Examples with IndexedInputGate org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate used on opensource projects

Example 1 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class StreamTask method restoreGates.

private CompletableFuture<Void> restoreGates() throws Exception {
    SequentialChannelStateReader reader = getEnvironment().getTaskStateManager().getSequentialChannelStateReader();
    reader.readOutputData(getEnvironment().getAllWriters(), !configuration.isGraphContainingLoops());
    operatorChain.initializeStateAndOpenOperators(createStreamTaskStateInitializer());
    IndexedInputGate[] inputGates = getEnvironment().getAllInputGates();
    channelIOExecutor.execute(() -> {
        try {
            reader.readInputData(inputGates);
        } catch (Exception e) {
            asyncExceptionHandler.handleAsyncException("Unable to read channel state", e);
        }
    });
    // We wait for all input channel state to recover before we go into RUNNING state, and thus
    // start checkpointing. If we implement incremental checkpointing of input channel state
    // we must make sure it supports CheckpointType#FULL_CHECKPOINT
    List<CompletableFuture<?>> recoveredFutures = new ArrayList<>(inputGates.length);
    for (InputGate inputGate : inputGates) {
        recoveredFutures.add(inputGate.getStateConsumedFuture());
        inputGate.getStateConsumedFuture().thenRun(() -> mainMailboxExecutor.execute(inputGate::requestPartitions, "Input gate request partitions"));
    }
    return CompletableFuture.allOf(recoveredFutures.toArray(new CompletableFuture[0])).thenRun(mailboxProcessor::suspend);
}

Example 2 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class InputProcessorUtilTest method testCreateCheckpointedMultipleInputGate.

@Test
public void testCreateCheckpointedMultipleInputGate() throws Exception {
    try (CloseableRegistry registry = new CloseableRegistry()) {
        MockEnvironment environment = new MockEnvironmentBuilder().build();
        MockStreamTask streamTask = new MockStreamTaskBuilder(environment).build();
        StreamConfig streamConfig = new StreamConfig(environment.getJobConfiguration());
        streamConfig.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE);
        streamConfig.setUnalignedCheckpointsEnabled(true);
        // First input gate has index larger than the second
        List<IndexedInputGate>[] inputGates = new List[] { Collections.singletonList(getGate(1, 4)), Collections.singletonList(getGate(0, 2)) };
        CheckpointBarrierHandler barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(streamTask, streamConfig, new TestSubtaskCheckpointCoordinator(new MockChannelStateWriter()), streamTask.getName(), inputGates, Collections.emptyList(), new SyncMailboxExecutor(), new TestProcessingTimeService());
        CheckpointedInputGate[] checkpointedMultipleInputGate = InputProcessorUtil.createCheckpointedMultipleInputGate(new SyncMailboxExecutor(), inputGates, environment.getMetricGroup().getIOMetricGroup(), barrierHandler, streamConfig);
        for (CheckpointedInputGate checkpointedInputGate : checkpointedMultipleInputGate) {
            registry.registerCloseable(checkpointedInputGate);
        }
        List<IndexedInputGate> allInputGates = Arrays.stream(inputGates).flatMap(gates -> gates.stream()).collect(Collectors.toList());
        for (IndexedInputGate inputGate : allInputGates) {
            for (int channelId = 0; channelId < inputGate.getNumberOfInputChannels(); channelId++) {
                barrierHandler.processBarrier(new CheckpointBarrier(1, 42, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(inputGate.getGateIndex(), channelId), false);
            }
        }
        assertTrue(barrierHandler.getAllBarriersReceivedFuture(1).isDone());
    }
}

Example 3 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class StreamTask method triggerUnfinishedChannelsCheckpoint.

private boolean triggerUnfinishedChannelsCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
    Optional<CheckpointBarrierHandler> checkpointBarrierHandler = getCheckpointBarrierHandler();
    checkState(checkpointBarrierHandler.isPresent(), "CheckpointBarrier should exist for tasks with network inputs.");
    CheckpointBarrier barrier = new CheckpointBarrier(checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions);
    for (IndexedInputGate inputGate : getEnvironment().getAllInputGates()) {
        if (!inputGate.isFinished()) {
            for (InputChannelInfo channelInfo : inputGate.getUnfinishedChannels()) {
                checkpointBarrierHandler.get().processBarrier(barrier, channelInfo, true);
            }
        }
    }
    return true;
}

Example 4 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class MultipleInputStreamTask method init.

@SuppressWarnings("rawtypes")
@Override
public void init() throws Exception {
    StreamConfig configuration = getConfiguration();
    ClassLoader userClassLoader = getUserCodeClassLoader();
    InputConfig[] inputs = configuration.getInputs(userClassLoader);
    WatermarkGauge[] watermarkGauges = new WatermarkGauge[inputs.length];
    for (int i = 0; i < inputs.length; i++) {
        watermarkGauges[i] = new WatermarkGauge();
        mainOperator.getMetricGroup().gauge(MetricNames.currentInputWatermarkName(i + 1), watermarkGauges[i]);
    }
    MinWatermarkGauge minInputWatermarkGauge = new MinWatermarkGauge(watermarkGauges);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
    List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
    // Those two number may differ for example when one of the inputs is a union. In that case
    // the number of logical network inputs is smaller compared to the number of inputs (input
    // gates)
    int numberOfNetworkInputs = configuration.getNumberOfNetworkInputs();
    ArrayList[] inputLists = new ArrayList[inputs.length];
    for (int i = 0; i < inputLists.length; i++) {
        inputLists[i] = new ArrayList<>();
    }
    for (int i = 0; i < numberOfNetworkInputs; i++) {
        int inputType = inEdges.get(i).getTypeNumber();
        IndexedInputGate reader = getEnvironment().getInputGate(i);
        inputLists[inputType - 1].add(reader);
    }
    ArrayList<ArrayList<?>> networkInputLists = new ArrayList<>();
    for (ArrayList<?> inputList : inputLists) {
        if (!inputList.isEmpty()) {
            networkInputLists.add(inputList);
        }
    }
    createInputProcessor(networkInputLists.toArray(new ArrayList[0]), inputs, watermarkGauges, (index) -> inEdges.get(index).getPartitioner());
    // wrap watermark gauge since registered metrics must be unique
    getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}

Example 5 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class AbstractTwoInputStreamTask method init.

@Override
public void init() throws Exception {
    StreamConfig configuration = getConfiguration();
    ClassLoader userClassLoader = getUserCodeClassLoader();
    int numberOfInputs = configuration.getNumberOfNetworkInputs();
    ArrayList<IndexedInputGate> inputList1 = new ArrayList<>();
    ArrayList<IndexedInputGate> inputList2 = new ArrayList<>();
    List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
    for (int i = 0; i < numberOfInputs; i++) {
        int inputType = inEdges.get(i).getTypeNumber();
        IndexedInputGate reader = getEnvironment().getInputGate(i);
        switch(inputType) {
            case 1:
                inputList1.add(reader);
                break;
            case 2:
                inputList2.add(reader);
                break;
            default:
                throw new RuntimeException("Invalid input type number: " + inputType);
        }
    }
    createInputProcessor(inputList1, inputList2, gateIndex -> inEdges.get(gateIndex).getPartitioner());
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_1_WATERMARK, input1WatermarkGauge);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_2_WATERMARK, input2WatermarkGauge);
    // wrap watermark gauge since registered metrics must be unique
    getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}