Examples with InputSpec org.apache.flink.table.runtime.operators.multipleinput.input.InputSpec used on opensource projects

Example 6 with InputSpec

use of org.apache.flink.table.runtime.operators.multipleinput.input.InputSpec in project flink by apache.

the class TableOperatorWrapperGeneratorTest method testDifferentParallelisms.

/**
 * Test for nodes with different parallelisms in a multiple input node.
 *
 * <pre>
 *
 *       source1  source2
 *         |        |
 *        calc1   calc2
 *           \    /
 * source3   union
 *      \      /
 *        join
 *
 * </pre>
 */
@Test
public void testDifferentParallelisms() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    Transformation<RowData> source1 = createSource(env, "source1");
    Transformation<RowData> source2 = createSource(env, "source2");
    Transformation<RowData> source3 = createSource(env, "source3");
    OneInputTransformation<RowData, RowData> calc1 = createOneInputTransform(source1, "calc1", InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    calc1.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, 1);
    calc1.setParallelism(100);
    OneInputTransformation<RowData, RowData> calc2 = createOneInputTransform(source2, "calc2", InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    calc2.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, 1);
    calc2.setParallelism(50);
    UnionTransformation<RowData> union = createUnionInputTransform("union1", calc1, calc2);
    TwoInputTransformation<RowData, RowData, RowData> join = createTwoInputTransform(union, source3, "join1", InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    join.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, 1);
    join.setParallelism(200);
    TableOperatorWrapperGenerator generator = new TableOperatorWrapperGenerator(Arrays.asList(source1, source2, source3), join, new int[] { 1, 1, 0 });
    generator.generate();
    TableOperatorWrapper<?> calcWrapper1 = createWrapper(calc1, 2, 1.0 / 3);
    TableOperatorWrapper<?> calcWrapper2 = createWrapper(calc2, 3, 1.0 / 3);
    TableOperatorWrapper<?> unionWrapper = createWrapper(union, 1);
    unionWrapper.addInput(calcWrapper1, 1);
    unionWrapper.addInput(calcWrapper2, 2);
    TableOperatorWrapper<?> outputWrapper = createWrapper(join, 0, 1.0 / 3);
    outputWrapper.addInput(unionWrapper, 2);
    assertEquals(Arrays.asList(Pair.of(source1, new InputSpec(1, 1, calcWrapper1, 1)), Pair.of(source2, new InputSpec(2, 1, calcWrapper2, 1)), Pair.of(source3, new InputSpec(3, 0, outputWrapper, 2))), generator.getInputTransformAndInputSpecPairs());
    assertEquals(Arrays.asList(new TableOperatorWrapper.Edge(calcWrapper1, unionWrapper, 1), new TableOperatorWrapper.Edge(calcWrapper2, unionWrapper, 2)), unionWrapper.getInputEdges());
    assertEquals(Collections.singletonList(new TableOperatorWrapper.Edge(unionWrapper, outputWrapper, 2)), outputWrapper.getInputEdges());
    assertEquals(outputWrapper, generator.getTailWrapper());
    assertEquals(3, generator.getManagedMemoryWeight());
    assertEquals(200, generator.getParallelism());
    assertEquals(-1, generator.getMaxParallelism());
    assertEquals(ResourceSpec.UNKNOWN, generator.getMinResources());
    assertEquals(ResourceSpec.UNKNOWN, generator.getPreferredResources());
}

Example 7 with InputSpec

use of org.apache.flink.table.runtime.operators.multipleinput.input.InputSpec in project flink by apache.

the class TableOperatorWrapperGeneratorTest method testWithUnion.

/**
 * Test for union nodes in a multiple input node.
 *
 * <pre>
 *
 *          source1  source2
 *               \    /
 *     source4   union1  source3
 *         \         \    /
 *         agg1      union2
 *            \      /
 * source5     join1
 *      \      /
 *       union3
 *
 * </pre>
 */
@Test
public void testWithUnion() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    Transformation<RowData> source1 = createSource(env, "source1");
    Transformation<RowData> source2 = createSource(env, "source2");
    Transformation<RowData> source3 = createSource(env, "source3");
    Transformation<RowData> source4 = createSource(env, "source4");
    Transformation<RowData> source5 = createSource(env, "source5");
    UnionTransformation<RowData> union1 = createUnionInputTransform("union1", source1, source2);
    UnionTransformation<RowData> union2 = createUnionInputTransform("union2", union1, source3);
    OneInputTransformation<RowData, RowData> agg1 = createOneInputTransform(source4, "agg1", InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    agg1.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, 1);
    TwoInputTransformation<RowData, RowData, RowData> join1 = createTwoInputTransform(agg1, union2, "join1", InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    join1.declareManagedMemoryUseCaseAtOperatorScope(ManagedMemoryUseCase.OPERATOR, 2);
    UnionTransformation<RowData> union3 = createUnionInputTransform("union3", source5, join1);
    TableOperatorWrapperGenerator generator = new TableOperatorWrapperGenerator(Arrays.asList(source1, source2, source3, source4, source5), union3, new int[] { 1, 1, 1, 0, 2 });
    generator.generate();
    TableOperatorWrapper<?> unionWrapper1 = createWrapper(union1, 4);
    TableOperatorWrapper<?> unionWrapper2 = createWrapper(union2, 3);
    unionWrapper2.addInput(unionWrapper1, 1);
    TableOperatorWrapper<?> aggWrapper1 = createWrapper(agg1, 2, 1.0 / 3);
    TableOperatorWrapper<?> joinWrapper1 = createWrapper(join1, 1, 2.0 / 3);
    joinWrapper1.addInput(aggWrapper1, 1);
    joinWrapper1.addInput(unionWrapper2, 2);
    TableOperatorWrapper<?> outputWrapper = createWrapper(union3, 0);
    outputWrapper.addInput(joinWrapper1, 1);
    assertEquals(Arrays.asList(Pair.of(source5, new InputSpec(1, 2, outputWrapper, 1)), Pair.of(source4, new InputSpec(2, 0, aggWrapper1, 1)), Pair.of(source1, new InputSpec(3, 1, unionWrapper1, 1)), Pair.of(source2, new InputSpec(4, 1, unionWrapper1, 1)), Pair.of(source3, new InputSpec(5, 1, unionWrapper2, 1))), generator.getInputTransformAndInputSpecPairs());
    assertEquals(outputWrapper, generator.getTailWrapper());
    assertEquals(3, generator.getManagedMemoryWeight());
    assertEquals(10, generator.getParallelism());
    assertEquals(-1, generator.getMaxParallelism());
    assertEquals(ResourceSpec.UNKNOWN, generator.getMinResources());
    assertEquals(ResourceSpec.UNKNOWN, generator.getPreferredResources());
}