Examples with SemanticProperties org.apache.flink.api.common.operators.SemanticProperties used on opensource projects

Example 6 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GroupReduceOperatorTest method testSemanticPropsWithKeySelector6.

@Test
public void testSemanticPropsWithKeySelector6() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).sortGroup(new DummyTestKeySelector(), Order.ASCENDING).reduceGroup(new DummyGroupReduceFunction3()).withForwardedFields("4->0;3;3->1;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 7).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 7).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 8).contains(0));
    assertTrue(semProps.getForwardingSourceField(0, 0) == 8);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 7);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 7);
    assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
    assertTrue(semProps.getReadFields(0) == null);
}

Example 7 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GroupReduceOperatorTest method testSemanticPropsWithKeySelector7.

@Test
public void testSemanticPropsWithKeySelector7() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduceGroup(new DummyGroupReduceFunction4());
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 2).contains(0));
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(semProps.getForwardingSourceField(0, 0) == 2);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 2) < 0);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
    assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
    assertTrue(semProps.getReadFields(0) == null);
}

Example 8 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GroupReduceOperatorTest method testSemanticPropsWithKeySelector3.

@Test
public void testSemanticPropsWithKeySelector3() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduceGroup(new DummyGroupReduceFunction2()).withForwardedFields("0->4;1;1->3;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
    assertTrue(semProps.getReadFields(0).size() == 3);
    assertTrue(semProps.getReadFields(0).contains(2));
    assertTrue(semProps.getReadFields(0).contains(5));
    assertTrue(semProps.getReadFields(0).contains(6));
}

Example 9 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class JoinOperatorTest method testSemanticPropsWithKeySelector2.

@Test
public void testSemanticPropsWithKeySelector2() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
    JoinOperator<?, ?, ?> joinOp = tupleDs1.join(tupleDs2).where(new DummyTestKeySelector()).equalTo(new DummyTestKeySelector()).with(new DummyTestJoinFunction2()).withForwardedFieldsFirst("2;4->0").withForwardedFieldsSecond("0->4;1;1->3");
    SemanticProperties semProps = joinOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
    assertTrue(semProps.getForwardingTargetFields(1, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(1, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(1, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(1, 2).contains(4));
    assertTrue(semProps.getForwardingTargetFields(1, 3).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(1, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(1, 3).contains(3));
    assertTrue(semProps.getForwardingTargetFields(1, 4).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(1, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(1, 6).size() == 0);
    assertTrue(semProps.getReadFields(0).size() == 3);
    assertTrue(semProps.getReadFields(0).contains(2));
    assertTrue(semProps.getReadFields(0).contains(3));
    assertTrue(semProps.getReadFields(0).contains(4));
    assertTrue(semProps.getReadFields(1) == null);
}

Example 10 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class CoGroupNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    DualInputSemanticProperties origProps = new DualInputSemanticProperties();
    // props for first input
    origProps.addForwardedField(0, 0, 1);
    origProps.addForwardedField(0, 2, 2);
    origProps.addForwardedField(0, 3, 4);
    origProps.addForwardedField(0, 6, 0);
    origProps.addReadFields(0, new FieldSet(0, 2, 4, 7));
    // props for second input
    origProps.addForwardedField(1, 1, 2);
    origProps.addForwardedField(1, 2, 8);
    origProps.addForwardedField(1, 3, 7);
    origProps.addForwardedField(1, 6, 6);
    origProps.addReadFields(1, new FieldSet(1, 3, 4));
    CoGroupOperatorBase<?, ?, ?, ?> op = mock(CoGroupOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getKeyColumns(1)).thenReturn(new int[] { 6, 3 });
    when(op.getParameters()).thenReturn(new Configuration());
    CoGroupNode node = new CoGroupNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    // check first input props
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    // check second input props
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 1).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).contains(6));
    assertTrue(filteredProps.getForwardingSourceField(1, 2) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 8) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 7) == 3);
    assertTrue(filteredProps.getForwardingSourceField(1, 6) == 6);
    assertTrue(filteredProps.getReadFields(1).size() == 3);
    assertTrue(filteredProps.getReadFields(1).contains(1));
    assertTrue(filteredProps.getReadFields(1).contains(3));
    assertTrue(filteredProps.getReadFields(1).contains(4));
}