Examples with OutputTag org.apache.flink.util.OutputTag used on opensource projects

Example 61 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class OperatorChain method createStreamOutput.

private RecordWriterOutput<OUT> createStreamOutput(RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter, StreamEdge edge, StreamConfig upStreamConfig, Environment taskEnvironment) {
    // OutputTag, return null if not sideOutput
    OutputTag sideOutputTag = edge.getOutputTag();
    TypeSerializer outSerializer;
    if (edge.getOutputTag() != null) {
        // side output
        outSerializer = upStreamConfig.getTypeSerializerSideOut(edge.getOutputTag(), taskEnvironment.getUserCodeClassLoader().asClassLoader());
    } else {
        // main output
        outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserCodeClassLoader().asClassLoader());
    }
    return closer.register(new RecordWriterOutput<OUT>(recordWriter, outSerializer, sideOutputTag, edge.supportsUnalignedCheckpoints()));
}

Example 62 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class OperatorChain method createChainedSources.

@SuppressWarnings("rawtypes")
private Map<StreamConfig.SourceInputConfig, ChainedSource> createChainedSources(StreamTask<OUT, OP> containingTask, StreamConfig.InputConfig[] configuredInputs, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, List<StreamOperatorWrapper<?, ?>> allOpWrappers) {
    if (Arrays.stream(configuredInputs).noneMatch(input -> input instanceof StreamConfig.SourceInputConfig)) {
        return Collections.emptyMap();
    }
    checkState(mainOperatorWrapper.getStreamOperator() instanceof MultipleInputStreamOperator, "Creating chained input is only supported with MultipleInputStreamOperator and MultipleInputStreamTask");
    Map<StreamConfig.SourceInputConfig, ChainedSource> chainedSourceInputs = new HashMap<>();
    MultipleInputStreamOperator<?> multipleInputOperator = (MultipleInputStreamOperator<?>) mainOperatorWrapper.getStreamOperator();
    List<Input> operatorInputs = multipleInputOperator.getInputs();
    int sourceInputGateIndex = Arrays.stream(containingTask.getEnvironment().getAllInputGates()).mapToInt(IndexedInputGate::getInputGateIndex).max().orElse(-1) + 1;
    for (int inputId = 0; inputId < configuredInputs.length; inputId++) {
        if (!(configuredInputs[inputId] instanceof StreamConfig.SourceInputConfig)) {
            continue;
        }
        StreamConfig.SourceInputConfig sourceInput = (StreamConfig.SourceInputConfig) configuredInputs[inputId];
        int sourceEdgeId = sourceInput.getInputEdge().getSourceId();
        StreamConfig sourceInputConfig = chainedConfigs.get(sourceEdgeId);
        OutputTag outputTag = sourceInput.getInputEdge().getOutputTag();
        WatermarkGaugeExposingOutput chainedSourceOutput = createChainedSourceOutput(containingTask, sourceInputConfig, userCodeClassloader, getFinishedOnRestoreInputOrDefault(operatorInputs.get(inputId)), multipleInputOperator.getMetricGroup(), outputTag);
        SourceOperator<?, ?> sourceOperator = (SourceOperator<?, ?>) createOperator(containingTask, sourceInputConfig, userCodeClassloader, (WatermarkGaugeExposingOutput<StreamRecord<OUT>>) chainedSourceOutput, allOpWrappers, true);
        chainedSourceInputs.put(sourceInput, new ChainedSource(chainedSourceOutput, this.isTaskDeployedAsFinished() ? new StreamTaskFinishedOnRestoreSourceInput<>(sourceOperator, sourceInputGateIndex++, inputId) : new StreamTaskSourceInput<>(sourceOperator, sourceInputGateIndex++, inputId)));
    }
    return chainedSourceInputs;
}

Example 63 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class IterateITCase method testmultipleHeadsTailsSimple.

@Test
public void testmultipleHeadsTailsSimple() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5).shuffle().map(noOpIntMap).name("ParallelizeMapShuffle");
    DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap).name("ParallelizeMapRebalance");
    IterativeStream<Integer> iter1 = source1.union(source2).iterate();
    DataStream<Integer> head1 = iter1.map(noOpIntMap).name("IterRebalanceMap").setParallelism(parallelism / 2);
    DataStream<Integer> head2 = iter1.map(noOpIntMap).name("IterForwardMap");
    DataStreamSink<Integer> head3 = iter1.map(noOpIntMap).setParallelism(parallelism / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
    DataStreamSink<Integer> head4 = iter1.map(noOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
    OutputTag<Integer> even = new OutputTag<Integer>("even") {
    };
    OutputTag<Integer> odd = new OutputTag<Integer>("odd") {
    };
    SingleOutputStreamOperator<Object> source3 = env.fromElements(1, 2, 3, 4, 5).map(noOpIntMap).name("EvenOddSourceMap").process(new ProcessFunction<Integer, Object>() {

        @Override
        public void processElement(Integer value, Context ctx, Collector<Object> out) throws Exception {
            if (value % 2 == 0) {
                ctx.output(even, value);
            } else {
                ctx.output(odd, value);
            }
        }
    });
    iter1.closeWith(source3.getSideOutput(even).union(head1.rebalance().map(noOpIntMap).broadcast(), head2.shuffle()));
    StreamGraph graph = env.getStreamGraph();
    JobGraph jg = graph.getJobGraph();
    assertEquals(1, graph.getIterationSourceSinkPairs().size());
    Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
    StreamNode itSource = sourceSinkPair.f0;
    StreamNode itSink = sourceSinkPair.f1;
    assertEquals(4, itSource.getOutEdges().size());
    assertEquals(3, itSink.getInEdges().size());
    assertEquals(itSource.getParallelism(), itSink.getParallelism());
    for (StreamEdge edge : itSource.getOutEdges()) {
        if (graph.getTargetVertex(edge).getOperatorName().equals("IterRebalanceMap")) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
        } else if (graph.getTargetVertex(edge).getOperatorName().equals("IterForwardMap")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
    }
    for (StreamEdge edge : itSink.getInEdges()) {
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) {
            assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
        }
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
        if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        }
    }
    // Test co-location
    JobVertex itSource1 = null;
    JobVertex itSink1 = null;
    for (JobVertex vertex : jg.getVertices()) {
        if (vertex.getName().contains("IterationSource")) {
            itSource1 = vertex;
        } else if (vertex.getName().contains("IterationSink")) {
            itSink1 = vertex;
        }
    }
    assertTrue(itSource1.getCoLocationGroup() != null);
    assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
}

Example 64 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class SideOutputITCase method testKeyedProcessFunctionSideOutput.

/**
 * Test keyed ProcessFunction side output.
 */
@Test
public void testKeyedProcessFunctionSideOutput() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.keyBy(new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }).process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    passThroughtStream.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 65 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class SideOutputITCase method testUnionOfTwoSideOutputs.

@Test
public void testUnionOfTwoSideOutputs() throws Exception {
    TestListResultSink<Integer> evensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUEvensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> evensUOddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUOddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> evensUEvensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUEvensExternalResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> input = env.fromElements(1, 2, 3, 4);
    OutputTag<Integer> oddTag = new OutputTag<Integer>("odds") {
    };
    OutputTag<Integer> evenTag = new OutputTag<Integer>("even") {
    };
    SingleOutputStreamOperator<Integer> passThroughStream = input.process(new ProcessFunction<Integer, Integer>() {

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            if (value % 2 != 0) {
                ctx.output(oddTag, value);
            } else {
                ctx.output(evenTag, value);
            }
            out.collect(value);
        }
    });
    DataStream<Integer> evens = passThroughStream.getSideOutput(evenTag);
    DataStream<Integer> odds = passThroughStream.getSideOutput(oddTag);
    evens.addSink(evensResultSink);
    odds.addSink(oddsResultSink);
    passThroughStream.addSink(resultSink);
    odds.union(evens).addSink(oddsUEvensResultSink);
    evens.union(odds).addSink(evensUOddsResultSink);
    odds.union(odds).addSink(oddsUOddsResultSink);
    evens.union(evens).addSink(evensUEvensResultSink);
    odds.union(env.fromElements(2, 4)).addSink(oddsUEvensExternalResultSink);
    env.execute();
    assertEquals(Arrays.asList(1, 3), oddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(2, 4), evensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), resultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), oddsUEvensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), evensUOddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 1, 3, 3), oddsUOddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(2, 2, 4, 4), evensUEvensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), oddsUEvensExternalResultSink.getSortedResult());
}