Examples with SemanticProperties org.apache.flink.api.common.operators.SemanticProperties used on opensource projects

Example 1 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GenericFlatTypePostPass method addSchemaToSchema.

private void addSchemaToSchema(T sourceSchema, T targetSchema, OptimizerNode optNode, int input) {
    try {
        for (Map.Entry<Integer, X> entry : sourceSchema) {
            Integer pos = entry.getKey();
            SemanticProperties sprops = optNode.getSemanticProperties();
            if (sprops != null && sprops.getForwardingTargetFields(input, pos) != null && sprops.getForwardingTargetFields(input, pos).contains(pos)) {
                targetSchema.addType(pos, entry.getValue());
            }
        }
    } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber() + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " + "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
    }
}

Example 2 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class ReduceOperatorTest method testSemanticPropsWithKeySelector4.

@Test
public void testSemanticPropsWithKeySelector4() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduce(new DummyReduceFunction4());
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 2).contains(0));
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(semProps.getForwardingSourceField(0, 0) == 2);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 2) < 0);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
    assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
    assertTrue(semProps.getReadFields(0) == null);
}

Example 3 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class ReduceOperatorTest method testSemanticPropsWithKeySelector2.

@Test
public void testSemanticPropsWithKeySelector2() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduce(new DummyReduceFunction2()).withForwardedFields("0->4;1;1->3;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
    assertTrue(semProps.getReadFields(0).size() == 3);
    assertTrue(semProps.getReadFields(0).contains(2));
    assertTrue(semProps.getReadFields(0).contains(5));
    assertTrue(semProps.getReadFields(0).contains(6));
}

Example 4 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class ReduceOperatorTest method testSemanticPropsWithKeySelector3.

@Test
public void testSemanticPropsWithKeySelector3() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduce(new DummyReduceFunction3()).withForwardedFields("4->0;3;3->1;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
    assertTrue(semProps.getForwardingSourceField(0, 0) == 6);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
    assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
    assertTrue(semProps.getReadFields(0) == null);
}

Example 5 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GroupReduceOperatorTest method testSemanticPropsWithKeySelector6.

@Test
public void testSemanticPropsWithKeySelector6() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).sortGroup(new DummyTestKeySelector(), Order.ASCENDING).reduceGroup(new DummyGroupReduceFunction3()).withForwardedFields("4->0;3;3->1;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 7).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 7).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 8).contains(0));
    assertTrue(semProps.getForwardingSourceField(0, 0) == 8);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 7);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 7);
    assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
    assertTrue(semProps.getReadFields(0) == null);
}