Search in sources :

Example 1 with MultipleInputStreamOperator

use of org.apache.flink.streaming.api.operators.MultipleInputStreamOperator in project flink by apache.

the class StreamMultipleInputProcessorFactory method create.

@SuppressWarnings({ "unchecked", "rawtypes" })
public static StreamMultipleInputProcessor create(TaskInvokable ownerTask, CheckpointedInputGate[] checkpointedInputGates, StreamConfig.InputConfig[] configuredInputs, IOManager ioManager, MemoryManager memoryManager, TaskIOMetricGroup ioMetricGroup, Counter mainOperatorRecordsIn, MultipleInputStreamOperator<?> mainOperator, WatermarkGauge[] inputWatermarkGauges, StreamConfig streamConfig, Configuration taskManagerConfig, Configuration jobConfig, ExecutionConfig executionConfig, ClassLoader userClassloader, OperatorChain<?, ?> operatorChain, InflightDataRescalingDescriptor inflightDataRescalingDescriptor, Function<Integer, StreamPartitioner<?>> gatePartitioners, TaskInfo taskInfo) {
    checkNotNull(operatorChain);
    List<Input> operatorInputs = mainOperator.getInputs();
    int inputsCount = operatorInputs.size();
    StreamOneInputProcessor<?>[] inputProcessors = new StreamOneInputProcessor[inputsCount];
    Counter networkRecordsIn = new SimpleCounter();
    ioMetricGroup.reuseRecordsInputCounter(networkRecordsIn);
    checkState(configuredInputs.length == inputsCount, "Number of configured inputs in StreamConfig [%s] doesn't match the main operator's number of inputs [%s]", configuredInputs.length, inputsCount);
    StreamTaskInput[] inputs = new StreamTaskInput[inputsCount];
    for (int i = 0; i < inputsCount; i++) {
        StreamConfig.InputConfig configuredInput = configuredInputs[i];
        if (configuredInput instanceof StreamConfig.NetworkInputConfig) {
            StreamConfig.NetworkInputConfig networkInput = (StreamConfig.NetworkInputConfig) configuredInput;
            inputs[i] = StreamTaskNetworkInputFactory.create(checkpointedInputGates[networkInput.getInputGateIndex()], networkInput.getTypeSerializer(), ioManager, new StatusWatermarkValve(checkpointedInputGates[networkInput.getInputGateIndex()].getNumberOfInputChannels()), i, inflightDataRescalingDescriptor, gatePartitioners, taskInfo);
        } else if (configuredInput instanceof StreamConfig.SourceInputConfig) {
            StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInput;
            inputs[i] = operatorChain.getSourceTaskInput(sourceInput);
        } else {
            throw new UnsupportedOperationException("Unknown input type: " + configuredInput);
        }
    }
    InputSelectable inputSelectable = mainOperator instanceof InputSelectable ? (InputSelectable) mainOperator : null;
    StreamConfig.InputConfig[] inputConfigs = streamConfig.getInputs(userClassloader);
    boolean anyRequiresSorting = Arrays.stream(inputConfigs).anyMatch(StreamConfig::requiresSorting);
    if (anyRequiresSorting) {
        if (inputSelectable != null) {
            throw new IllegalStateException("The InputSelectable interface is not supported with sorting inputs");
        }
        StreamTaskInput[] sortingInputs = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> inputs[idx]).toArray(StreamTaskInput[]::new);
        KeySelector[] sortingInputKeySelectors = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> streamConfig.getStatePartitioner(idx, userClassloader)).toArray(KeySelector[]::new);
        TypeSerializer[] sortingInputKeySerializers = IntStream.range(0, inputsCount).filter(idx -> requiresSorting(inputConfigs[idx])).mapToObj(idx -> streamConfig.getTypeSerializerIn(idx, userClassloader)).toArray(TypeSerializer[]::new);
        StreamTaskInput[] passThroughInputs = IntStream.range(0, inputsCount).filter(idx -> !requiresSorting(inputConfigs[idx])).mapToObj(idx -> inputs[idx]).toArray(StreamTaskInput[]::new);
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(ownerTask, sortingInputs, sortingInputKeySelectors, sortingInputKeySerializers, streamConfig.getStateKeySerializer(userClassloader), passThroughInputs, memoryManager, ioManager, executionConfig.isObjectReuseEnabled(), streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(ManagedMemoryUseCase.OPERATOR, taskManagerConfig, userClassloader), jobConfig, executionConfig);
        StreamTaskInput<?>[] sortedInputs = selectableSortingInputs.getSortedInputs();
        StreamTaskInput<?>[] passedThroughInputs = selectableSortingInputs.getPassThroughInputs();
        int sortedIndex = 0;
        int passThroughIndex = 0;
        for (int i = 0; i < inputs.length; i++) {
            if (requiresSorting(inputConfigs[i])) {
                inputs[i] = sortedInputs[sortedIndex];
                sortedIndex++;
            } else {
                inputs[i] = passedThroughInputs[passThroughIndex];
                passThroughIndex++;
            }
        }
        inputSelectable = selectableSortingInputs.getInputSelectable();
    }
    for (int i = 0; i < inputsCount; i++) {
        StreamConfig.InputConfig configuredInput = configuredInputs[i];
        if (configuredInput instanceof StreamConfig.NetworkInputConfig) {
            StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(operatorChain.getFinishedOnRestoreInputOrDefault(operatorInputs.get(i)), inputWatermarkGauges[i], mainOperatorRecordsIn, networkRecordsIn);
            inputProcessors[i] = new StreamOneInputProcessor(inputs[i], dataOutput, operatorChain);
        } else if (configuredInput instanceof StreamConfig.SourceInputConfig) {
            StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInput;
            OperatorChain.ChainedSource chainedSource = operatorChain.getChainedSource(sourceInput);
            inputProcessors[i] = new StreamOneInputProcessor(inputs[i], new StreamTaskSourceOutput(chainedSource.getSourceOutput(), inputWatermarkGauges[i], chainedSource.getSourceTaskInput().getOperator().getSourceMetricGroup()), operatorChain);
        } else {
            throw new UnsupportedOperationException("Unknown input type: " + configuredInput);
        }
    }
    return new StreamMultipleInputProcessor(new MultipleInputSelectionHandler(inputSelectable, inputsCount), inputProcessors);
}
Also used : IntStream(java.util.stream.IntStream) TaskIOMetricGroup(org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup) Arrays(java.util.Arrays) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) InputSelectable(org.apache.flink.streaming.api.operators.InputSelectable) TaskInvokable(org.apache.flink.runtime.jobgraph.tasks.TaskInvokable) MemoryManager(org.apache.flink.runtime.memory.MemoryManager) CheckpointedInputGate(org.apache.flink.streaming.runtime.io.checkpointing.CheckpointedInputGate) StreamConfig.requiresSorting(org.apache.flink.streaming.api.graph.StreamConfig.requiresSorting) IOManager(org.apache.flink.runtime.io.disk.iomanager.IOManager) SelectableSortingInputs(org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput.SelectableSortingInputs) Watermark(org.apache.flink.streaming.api.watermark.Watermark) Function(java.util.function.Function) InflightDataRescalingDescriptor(org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor) StreamPartitioner(org.apache.flink.streaming.runtime.partitioner.StreamPartitioner) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) ManagedMemoryUseCase(org.apache.flink.core.memory.ManagedMemoryUseCase) SourceOperatorStreamTask(org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask) SimpleCounter(org.apache.flink.metrics.SimpleCounter) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) WatermarkGaugeExposingOutput(org.apache.flink.streaming.runtime.tasks.WatermarkGaugeExposingOutput) WatermarkStatus(org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus) Preconditions.checkState(org.apache.flink.util.Preconditions.checkState) StatusWatermarkValve(org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) KeySelector(org.apache.flink.api.java.functions.KeySelector) Configuration(org.apache.flink.configuration.Configuration) TaskInfo(org.apache.flink.api.common.TaskInfo) MultipleInputStreamOperator(org.apache.flink.streaming.api.operators.MultipleInputStreamOperator) InternalSourceReaderMetricGroup(org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup) List(java.util.List) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) OperatorChain(org.apache.flink.streaming.runtime.tasks.OperatorChain) Internal(org.apache.flink.annotation.Internal) MultiInputSortingDataInput(org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput) LatencyMarker(org.apache.flink.streaming.runtime.streamrecord.LatencyMarker) Counter(org.apache.flink.metrics.Counter) WatermarkGauge(org.apache.flink.streaming.runtime.metrics.WatermarkGauge) Input(org.apache.flink.streaming.api.operators.Input) InputSelectable(org.apache.flink.streaming.api.operators.InputSelectable) KeySelector(org.apache.flink.api.java.functions.KeySelector) SelectableSortingInputs(org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput.SelectableSortingInputs) MultiInputSortingDataInput(org.apache.flink.streaming.api.operators.sort.MultiInputSortingDataInput) Input(org.apache.flink.streaming.api.operators.Input) SimpleCounter(org.apache.flink.metrics.SimpleCounter) Counter(org.apache.flink.metrics.Counter) SimpleCounter(org.apache.flink.metrics.SimpleCounter) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) StatusWatermarkValve(org.apache.flink.streaming.runtime.watermarkstatus.StatusWatermarkValve)

Example 2 with MultipleInputStreamOperator

use of org.apache.flink.streaming.api.operators.MultipleInputStreamOperator in project flink by apache.

the class OperatorChain method createChainedSources.

@SuppressWarnings("rawtypes")
private Map<StreamConfig.SourceInputConfig, ChainedSource> createChainedSources(StreamTask<OUT, OP> containingTask, StreamConfig.InputConfig[] configuredInputs, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, List<StreamOperatorWrapper<?, ?>> allOpWrappers) {
    if (Arrays.stream(configuredInputs).noneMatch(input -> input instanceof StreamConfig.SourceInputConfig)) {
        return Collections.emptyMap();
    }
    checkState(mainOperatorWrapper.getStreamOperator() instanceof MultipleInputStreamOperator, "Creating chained input is only supported with MultipleInputStreamOperator and MultipleInputStreamTask");
    Map<StreamConfig.SourceInputConfig, ChainedSource> chainedSourceInputs = new HashMap<>();
    MultipleInputStreamOperator<?> multipleInputOperator = (MultipleInputStreamOperator<?>) mainOperatorWrapper.getStreamOperator();
    List<Input> operatorInputs = multipleInputOperator.getInputs();
    int sourceInputGateIndex = Arrays.stream(containingTask.getEnvironment().getAllInputGates()).mapToInt(IndexedInputGate::getInputGateIndex).max().orElse(-1) + 1;
    for (int inputId = 0; inputId < configuredInputs.length; inputId++) {
        if (!(configuredInputs[inputId] instanceof StreamConfig.SourceInputConfig)) {
            continue;
        }
        StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInputs[inputId];
        int sourceEdgeId = sourceInput.getInputEdge().getSourceId();
        StreamConfig sourceInputConfig = chainedConfigs.get(sourceEdgeId);
        OutputTag outputTag = sourceInput.getInputEdge().getOutputTag();
        WatermarkGaugeExposingOutput chainedSourceOutput = createChainedSourceOutput(containingTask, sourceInputConfig, userCodeClassloader, getFinishedOnRestoreInputOrDefault(operatorInputs.get(inputId)), multipleInputOperator.getMetricGroup(), outputTag);
        SourceOperator<?, ?> sourceOperator = (SourceOperator<?, ?>) createOperator(containingTask, sourceInputConfig, userCodeClassloader, (WatermarkGaugeExposingOutput<StreamRecord<OUT>>) chainedSourceOutput, allOpWrappers, true);
        chainedSourceInputs.put(sourceInput, new ChainedSource(chainedSourceOutput, this.isTaskDeployedAsFinished() ? new StreamTaskFinishedOnRestoreSourceInput<>(sourceOperator, sourceInputGateIndex++, inputId) : new StreamTaskSourceInput<>(sourceOperator, sourceInputGateIndex++, inputId)));
    }
    return chainedSourceInputs;
}
Also used : HashMap(java.util.HashMap) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) BoundedMultiInput(org.apache.flink.streaming.api.operators.BoundedMultiInput) StreamTaskSourceInput(org.apache.flink.streaming.runtime.io.StreamTaskSourceInput) Input(org.apache.flink.streaming.api.operators.Input) MultipleInputStreamOperator(org.apache.flink.streaming.api.operators.MultipleInputStreamOperator) SourceOperator(org.apache.flink.streaming.api.operators.SourceOperator) OutputTag(org.apache.flink.util.OutputTag)

Aggregations

StreamConfig (org.apache.flink.streaming.api.graph.StreamConfig)2 Input (org.apache.flink.streaming.api.operators.Input)2 MultipleInputStreamOperator (org.apache.flink.streaming.api.operators.MultipleInputStreamOperator)2 Arrays (java.util.Arrays)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Function (java.util.function.Function)1 IntStream (java.util.stream.IntStream)1 Internal (org.apache.flink.annotation.Internal)1 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)1 TaskInfo (org.apache.flink.api.common.TaskInfo)1 TypeSerializer (org.apache.flink.api.common.typeutils.TypeSerializer)1 KeySelector (org.apache.flink.api.java.functions.KeySelector)1 Configuration (org.apache.flink.configuration.Configuration)1 ManagedMemoryUseCase (org.apache.flink.core.memory.ManagedMemoryUseCase)1 Counter (org.apache.flink.metrics.Counter)1 SimpleCounter (org.apache.flink.metrics.SimpleCounter)1 InflightDataRescalingDescriptor (org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor)1 IOManager (org.apache.flink.runtime.io.disk.iomanager.IOManager)1 TaskInvokable (org.apache.flink.runtime.jobgraph.tasks.TaskInvokable)1