Examples with SelectableSortingInputs org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput.SelectableSortingInputs used on opensource projects

Example 6 with SelectableSortingInputs

use of org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput.SelectableSortingInputs in project flink by apache.

the class StreamTwoInputProcessorFactory method create.

public static <IN1, IN2> StreamMultipleInputProcessor create(TaskInvokable ownerTask, CheckpointedInputGate[] checkpointedInputGates, IOManager ioManager, MemoryManager memoryManager, TaskIOMetricGroup taskIOMetricGroup, TwoInputStreamOperator<IN1, IN2, ?> streamOperator, WatermarkGauge input1WatermarkGauge, WatermarkGauge input2WatermarkGauge, OperatorChain<?, ?> operatorChain, StreamConfig streamConfig, Configuration taskManagerConfig, Configuration jobConfig, ExecutionConfig executionConfig, ClassLoader userClassloader, Counter numRecordsIn, InflightDataRescalingDescriptor inflightDataRescalingDescriptor, Function<Integer, StreamPartitioner<?>> gatePartitioners, TaskInfo taskInfo) {
    checkNotNull(operatorChain);
    taskIOMetricGroup.reuseRecordsInputCounter(numRecordsIn);
    TypeSerializer<IN1> typeSerializer1 = streamConfig.getTypeSerializerIn(0, userClassloader);
    StreamTaskInput<IN1> input1 = StreamTaskNetworkInputFactory.create(checkpointedInputGates[0], typeSerializer1, ioManager, new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels()), 0, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
    TypeSerializer<IN2> typeSerializer2 = streamConfig.getTypeSerializerIn(1, userClassloader);
    StreamTaskInput<IN2> input2 = StreamTaskNetworkInputFactory.create(checkpointedInputGates[1], typeSerializer2, ioManager, new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels()), 1, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
    InputSelectable inputSelectable = streamOperator instanceof InputSelectable ? (InputSelectable) streamOperator : null;
    // this is a bit verbose because we're manually handling input1 and input2
    // TODO: extract method
    StreamConfig.InputConfig[] inputConfigs = streamConfig.getInputs(userClassloader);
    boolean input1IsSorted = requiresSorting(inputConfigs[0]);
    boolean input2IsSorted = requiresSorting(inputConfigs[1]);
    if (input1IsSorted || input2IsSorted) {
        if (inputSelectable != null) {
            throw new IllegalStateException("The InputSelectable interface is not supported with sorting inputs");
        }
        List<StreamTaskInput<?>> sortedTaskInputs = new ArrayList<>();
        List<KeySelector<?, ?>> keySelectors = new ArrayList<>();
        List<StreamTaskInput<?>> passThroughTaskInputs = new ArrayList<>();
        if (input1IsSorted) {
            sortedTaskInputs.add(input1);
            keySelectors.add(streamConfig.getStatePartitioner(0, userClassloader));
        } else {
            passThroughTaskInputs.add(input1);
        }
        if (input2IsSorted) {
            sortedTaskInputs.add(input2);
            keySelectors.add(streamConfig.getStatePartitioner(1, userClassloader));
        } else {
            passThroughTaskInputs.add(input2);
        }
        @SuppressWarnings("unchecked") SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(ownerTask, sortedTaskInputs.toArray(new StreamTaskInput[0]), keySelectors.toArray(new KeySelector[0]), new TypeSerializer[] { typeSerializer1, typeSerializer2 }, streamConfig.getStateKeySerializer(userClassloader), passThroughTaskInputs.toArray(new StreamTaskInput[0]), memoryManager, ioManager, executionConfig.isObjectReuseEnabled(), streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(ManagedMemoryUseCase.OPERATOR, taskManagerConfig, userClassloader), jobConfig, executionConfig);
        inputSelectable = selectableSortingInputs.getInputSelectable();
        StreamTaskInput<?>[] sortedInputs = selectableSortingInputs.getSortedInputs();
        StreamTaskInput<?>[] passThroughInputs = selectableSortingInputs.getPassThroughInputs();
        if (input1IsSorted) {
            input1 = toTypedInput(sortedInputs[0]);
        } else {
            input1 = toTypedInput(passThroughInputs[0]);
        }
        if (input2IsSorted) {
            input2 = toTypedInput(sortedInputs[sortedInputs.length - 1]);
        } else {
            input2 = toTypedInput(passThroughInputs[passThroughInputs.length - 1]);
        }
    }
    @Nullable FinishedOnRestoreWatermarkBypass watermarkBypass = operatorChain.isTaskDeployedAsFinished() ? new FinishedOnRestoreWatermarkBypass(operatorChain.getStreamOutputs()) : null;
    StreamTaskNetworkOutput<IN1> output1 = new StreamTaskNetworkOutput<>(streamOperator, record -> processRecord1(record, streamOperator), input1WatermarkGauge, 0, numRecordsIn, watermarkBypass);
    StreamOneInputProcessor<IN1> processor1 = new StreamOneInputProcessor<>(input1, output1, operatorChain);
    StreamTaskNetworkOutput<IN2> output2 = new StreamTaskNetworkOutput<>(streamOperator, record -> processRecord2(record, streamOperator), input2WatermarkGauge, 1, numRecordsIn, watermarkBypass);
    StreamOneInputProcessor<IN2> processor2 = new StreamOneInputProcessor<>(input2, output2, operatorChain);
    return new StreamMultipleInputProcessor(new MultipleInputSelectionHandler(inputSelectable, 2), new StreamOneInputProcessor[] { processor1, processor2 });
}