Examples with StreamEdge org.apache.flink.streaming.api.graph.StreamEdge used on opensource projects

Example 16 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class MultipleInputStreamTask method init.

@SuppressWarnings("rawtypes")
@Override
public void init() throws Exception {
    StreamConfig configuration = getConfiguration();
    ClassLoader userClassLoader = getUserCodeClassLoader();
    InputConfig[] inputs = configuration.getInputs(userClassLoader);
    WatermarkGauge[] watermarkGauges = new WatermarkGauge[inputs.length];
    for (int i = 0; i < inputs.length; i++) {
        watermarkGauges[i] = new WatermarkGauge();
        mainOperator.getMetricGroup().gauge(MetricNames.currentInputWatermarkName(i + 1), watermarkGauges[i]);
    }
    MinWatermarkGauge minInputWatermarkGauge = new MinWatermarkGauge(watermarkGauges);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
    List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
    // Those two number may differ for example when one of the inputs is a union. In that case
    // the number of logical network inputs is smaller compared to the number of inputs (input
    // gates)
    int numberOfNetworkInputs = configuration.getNumberOfNetworkInputs();
    ArrayList[] inputLists = new ArrayList[inputs.length];
    for (int i = 0; i < inputLists.length; i++) {
        inputLists[i] = new ArrayList<>();
    }
    for (int i = 0; i < numberOfNetworkInputs; i++) {
        int inputType = inEdges.get(i).getTypeNumber();
        IndexedInputGate reader = getEnvironment().getInputGate(i);
        inputLists[inputType - 1].add(reader);
    }
    ArrayList<ArrayList<?>> networkInputLists = new ArrayList<>();
    for (ArrayList<?> inputList : inputLists) {
        if (!inputList.isEmpty()) {
            networkInputLists.add(inputList);
        }
    }
    createInputProcessor(networkInputLists.toArray(new ArrayList[0]), inputs, watermarkGauges, (index) -> inEdges.get(index).getPartitioner());
    // wrap watermark gauge since registered metrics must be unique
    getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}

Example 17 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class OperatorChain method createChainOutputs.

// ------------------------------------------------------------------------
// initialization utilities
// ------------------------------------------------------------------------
private void createChainOutputs(List<StreamEdge> outEdgesInOrder, RecordWriterDelegate<SerializationDelegate<StreamRecord<OUT>>> recordWriterDelegate, Map<Integer, StreamConfig> chainedConfigs, StreamTask<OUT, OP> containingTask, Map<StreamEdge, RecordWriterOutput<?>> streamOutputMap) {
    for (int i = 0; i < outEdgesInOrder.size(); i++) {
        StreamEdge outEdge = outEdgesInOrder.get(i);
        RecordWriterOutput<?> streamOutput = createStreamOutput(recordWriterDelegate.getRecordWriter(i), outEdge, chainedConfigs.get(outEdge.getSourceId()), containingTask.getEnvironment());
        this.streamOutputs[i] = streamOutput;
        streamOutputMap.put(outEdge, streamOutput);
    }
}

Example 18 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class OperatorChain method createOutputCollector.

private <T> WatermarkGaugeExposingOutput<StreamRecord<T>> createOutputCollector(StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperatorWrapper<?, ?>> allOperatorWrappers, MailboxExecutorFactory mailboxExecutorFactory) {
    List<Tuple2<WatermarkGaugeExposingOutput<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
    // create collectors for the network outputs
    for (StreamEdge outputEdge : operatorConfig.getNonChainedOutputs(userCodeClassloader)) {
        @SuppressWarnings("unchecked") RecordWriterOutput<T> output = (RecordWriterOutput<T>) streamOutputs.get(outputEdge);
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    // Create collectors for the chained outputs
    for (StreamEdge outputEdge : operatorConfig.getChainedOutputs(userCodeClassloader)) {
        int outputId = outputEdge.getTargetId();
        StreamConfig chainedOpConfig = chainedConfigs.get(outputId);
        WatermarkGaugeExposingOutput<StreamRecord<T>> output = createOperatorChain(containingTask, chainedOpConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperatorWrappers, outputEdge.getOutputTag(), mailboxExecutorFactory);
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    if (allOutputs.size() == 1) {
        return allOutputs.get(0).f0;
    } else {
        // send to N outputs. Note that this includes the special case
        // of sending to zero outputs
        @SuppressWarnings({ "unchecked" }) Output<StreamRecord<T>>[] asArray = new Output[allOutputs.size()];
        for (int i = 0; i < allOutputs.size(); i++) {
            asArray[i] = allOutputs.get(i).f0;
        }
        // otherwise multi-chaining would not work correctly.
        if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
            return closer.register(new CopyingBroadcastingOutputCollector<>(asArray));
        } else {
            return closer.register(new BroadcastingOutputCollector<>(asArray));
        }
    }
}

Example 19 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class AbstractTwoInputStreamTask method init.

@Override
public void init() throws Exception {
    StreamConfig configuration = getConfiguration();
    ClassLoader userClassLoader = getUserCodeClassLoader();
    int numberOfInputs = configuration.getNumberOfNetworkInputs();
    ArrayList<IndexedInputGate> inputList1 = new ArrayList<>();
    ArrayList<IndexedInputGate> inputList2 = new ArrayList<>();
    List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
    for (int i = 0; i < numberOfInputs; i++) {
        int inputType = inEdges.get(i).getTypeNumber();
        IndexedInputGate reader = getEnvironment().getInputGate(i);
        switch(inputType) {
            case 1:
                inputList1.add(reader);
                break;
            case 2:
                inputList2.add(reader);
                break;
            default:
                throw new RuntimeException("Invalid input type number: " + inputType);
        }
    }
    createInputProcessor(inputList1, inputList2, gateIndex -> inEdges.get(gateIndex).getPartitioner());
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_1_WATERMARK, input1WatermarkGauge);
    mainOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_2_WATERMARK, input2WatermarkGauge);
    // wrap watermark gauge since registered metrics must be unique
    getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}

Example 20 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class StreamTaskTestHarness method setupOutputForSingletonOperatorChain.

/**
 * Users of the test harness can call this utility method to setup the stream config if there
 * will only be a single operator to be tested. The method will setup the outgoing network
 * connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the
 * harness, please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
    Preconditions.checkState(!setupCalled, "This harness was already setup.");
    setupCalled = true;
    streamConfig.setChainStart();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
    streamConfig.setNumberOfOutputs(1);
    streamConfig.setTypeSerializerOut(outputSerializer);
    streamConfig.setVertexID(0);
    streamConfig.setOperatorID(new OperatorID(4711L, 123L));
    StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {

        private static final long serialVersionUID = 1L;
    };
    List<StreamEdge> outEdgesInOrder = new LinkedList<>();
    StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", SourceStreamTask.class);
    StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", SourceStreamTask.class);
    outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new BroadcastPartitioner<>(), null));
    streamConfig.setOutEdgesInOrder(outEdgesInOrder);
    streamConfig.setNonChainedOutputs(outEdgesInOrder);
}