Examples with DualInputSemanticProperties org.apache.flink.api.common.operators.DualInputSemanticProperties used on opensource projects

Example 1 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testDualGroupingPreserved.

@Test
public void testDualGroupingPreserved() {
    DualInputSemanticProperties dprops = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[] { "1->0;3;2->4" }, new String[] { "0->7;1" }, null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
    RequestedLocalProperties lprops1 = new RequestedLocalProperties();
    lprops1.setGroupedFields(new FieldSet(0, 3, 4));
    RequestedLocalProperties lprops2 = new RequestedLocalProperties();
    lprops2.setGroupedFields(new FieldSet(7, 1));
    RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
    RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);
    assertNotNull(filtered1);
    assertNotNull(filtered1.getGroupedFields());
    assertEquals(3, filtered1.getGroupedFields().size());
    assertTrue(filtered1.getGroupedFields().contains(1));
    assertTrue(filtered1.getGroupedFields().contains(2));
    assertTrue(filtered1.getGroupedFields().contains(3));
    assertNull(filtered1.getOrdering());
    assertNotNull(filtered2);
    assertNotNull(filtered2.getGroupedFields());
    assertEquals(2, filtered2.getGroupedFields().size());
    assertTrue(filtered2.getGroupedFields().contains(0));
    assertTrue(filtered2.getGroupedFields().contains(1));
    assertNull(filtered2.getOrdering());
}

Example 2 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class CoGroupNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    DualInputSemanticProperties origProps = new DualInputSemanticProperties();
    // props for first input
    origProps.addForwardedField(0, 0, 1);
    origProps.addForwardedField(0, 2, 2);
    origProps.addForwardedField(0, 3, 4);
    origProps.addForwardedField(0, 6, 0);
    origProps.addReadFields(0, new FieldSet(0, 2, 4, 7));
    // props for second input
    origProps.addForwardedField(1, 1, 2);
    origProps.addForwardedField(1, 2, 8);
    origProps.addForwardedField(1, 3, 7);
    origProps.addForwardedField(1, 6, 6);
    origProps.addReadFields(1, new FieldSet(1, 3, 4));
    CoGroupOperatorBase<?, ?, ?, ?> op = mock(CoGroupOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getKeyColumns(1)).thenReturn(new int[] { 6, 3 });
    when(op.getParameters()).thenReturn(new Configuration());
    CoGroupNode node = new CoGroupNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    // check first input props
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    // check second input props
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 1).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).contains(6));
    assertTrue(filteredProps.getForwardingSourceField(1, 2) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 8) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 7) == 3);
    assertTrue(filteredProps.getForwardingSourceField(1, 6) == 6);
    assertTrue(filteredProps.getReadFields(1).size() == 3);
    assertTrue(filteredProps.getReadFields(1).contains(1));
    assertTrue(filteredProps.getReadFields(1).contains(3));
    assertTrue(filteredProps.getReadFields(1).contains(4));
}

Example 3 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtilTest method testNonForwardedDualInvalidTypes1.

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedDualInvalidTypes1() {
    String[] nonNorwardedFieldsFirst = { "f1" };
    DualInputSemanticProperties dsp = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null, nonNorwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example 4 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtilTest method testForwardedDual.

// --------------------------------------------------------------------------------------------
// Two Inputs
// --------------------------------------------------------------------------------------------
@Test
public void testForwardedDual() {
    String[] forwardedFieldsFirst = { "f1->f2; f2->f3" };
    String[] forwardedFieldsSecond = { "f1->f1; f2->f0" };
    DualInputSemanticProperties dsp = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, fourIntTupleType, fourIntTupleType);
    assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2));
    assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3));
    assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
    assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0));
    assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0);
    forwardedFieldsFirst[0] = "f1->f0;f3->f1";
    forwardedFieldsSecond[0] = "*->f2.*";
    dsp = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, fourIntTupleType, pojoType, pojoInTupleType);
    assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 1).contains(0));
    assertTrue(dsp.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(dsp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(dsp.getForwardingTargetFields(1, 1).contains(3));
    assertTrue(dsp.getForwardingTargetFields(1, 2).contains(4));
    assertTrue(dsp.getForwardingTargetFields(1, 3).contains(5));
    forwardedFieldsFirst[0] = "f1.f0.f2->int1; f2->pojo1.int3";
    forwardedFieldsSecond[0] = "string1; int2->pojo1.int1; int1->pojo1.int2";
    dsp = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null, null, null, null, deepNestedTupleType, pojoType, nestedPojoType);
    assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 3).contains(0));
    assertTrue(dsp.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 6).contains(3));
    assertTrue(dsp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
    assertTrue(dsp.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(1, 3).contains(5));
    String[] forwardedFieldsFirst2 = { "f1.f0.f2->int1", "f2->pojo1.int3" };
    String[] forwardedFieldsSecond2 = { "string1", "int2->pojo1.int1", "int1->pojo1.int2" };
    dsp = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst2, forwardedFieldsSecond2, null, null, null, null, deepNestedTupleType, pojoType, nestedPojoType);
    assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 3).contains(0));
    assertTrue(dsp.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(0, 6).contains(3));
    assertTrue(dsp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
    assertTrue(dsp.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(dsp.getForwardingTargetFields(1, 3).contains(5));
}

Example 5 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtilTest method testDualProjectionProperties.

@Test
public void testDualProjectionProperties() {
    int[] pMap = new int[] { 4, 2, 0, 1, 3, 4 };
    boolean[] iMap = new boolean[] { true, true, false, true, false, false };
    DualInputSemanticProperties sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, fiveIntTupleType, fiveIntTupleType);
    assertTrue(sp.getForwardingTargetFields(0, 4).contains(0));
    assertTrue(sp.getForwardingTargetFields(0, 2).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(sp.getForwardingTargetFields(0, 1).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 3).contains(4));
    assertTrue(sp.getForwardingTargetFields(1, 4).contains(5));
    pMap = new int[] { 4, 2, 0, 4, 0, 1 };
    iMap = new boolean[] { true, true, false, true, false, false };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, fiveIntTupleType, fiveIntTupleType);
    assertTrue(sp.getForwardingTargetFields(0, 4).size() == 2);
    assertTrue(sp.getForwardingTargetFields(1, 0).size() == 2);
    assertTrue(sp.getForwardingTargetFields(0, 4).contains(0));
    assertTrue(sp.getForwardingTargetFields(0, 2).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(sp.getForwardingTargetFields(0, 4).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(4));
    assertTrue(sp.getForwardingTargetFields(1, 1).contains(5));
    pMap = new int[] { 2, 1, 0, 1 };
    iMap = new boolean[] { false, false, true, true };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, nestedTupleType, threeIntTupleType);
    assertTrue(sp.getForwardingTargetFields(1, 2).contains(0));
    assertTrue(sp.getForwardingTargetFields(1, 1).contains(1));
    assertTrue(sp.getForwardingTargetFields(0, 0).contains(2));
    assertTrue(sp.getForwardingTargetFields(0, 1).contains(3));
    assertTrue(sp.getForwardingTargetFields(0, 2).contains(4));
    assertTrue(sp.getForwardingTargetFields(0, 3).contains(5));
    pMap = new int[] { 1, 0, 0 };
    iMap = new boolean[] { false, false, true };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, nestedTupleType, deepNestedTupleType);
    assertTrue(sp.getForwardingTargetFields(1, 1).contains(0));
    assertTrue(sp.getForwardingTargetFields(1, 2).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 3).contains(2));
    assertTrue(sp.getForwardingTargetFields(1, 4).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 5).contains(4));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(5));
    assertTrue(sp.getForwardingTargetFields(0, 0).contains(6));
    assertTrue(sp.getForwardingTargetFields(0, 1).contains(7));
    assertTrue(sp.getForwardingTargetFields(0, 2).contains(8));
    pMap = new int[] { 4, 2, 1, 0 };
    iMap = new boolean[] { true, false, true, false };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, fiveIntTupleType, pojoInTupleType);
    assertTrue(sp.getForwardingTargetFields(0, 4).contains(0));
    assertTrue(sp.getForwardingTargetFields(1, 2).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 3).contains(2));
    assertTrue(sp.getForwardingTargetFields(1, 4).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 5).contains(4));
    assertTrue(sp.getForwardingTargetFields(0, 1).contains(5));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(6));
    pMap = new int[] { 2, 3, -1, 0 };
    iMap = new boolean[] { true, true, false, true };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, fiveIntTupleType, intType);
    assertTrue(sp.getForwardingTargetFields(0, 2).contains(0));
    assertTrue(sp.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(2));
    assertTrue(sp.getForwardingTargetFields(0, 0).contains(3));
    pMap = new int[] { -1, -1 };
    iMap = new boolean[] { false, true };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, intType, nestedPojoType);
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(0));
    assertTrue(sp.getForwardingTargetFields(1, 1).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 2).contains(2));
    assertTrue(sp.getForwardingTargetFields(1, 3).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 4).contains(4));
    assertTrue(sp.getForwardingTargetFields(1, 5).contains(5));
    assertTrue(sp.getForwardingTargetFields(0, 0).contains(6));
    pMap = new int[] { -1, -1 };
    iMap = new boolean[] { true, false };
    sp = SemanticPropUtil.createProjectionPropertiesDual(pMap, iMap, intType, nestedPojoType);
    assertTrue(sp.getForwardingTargetFields(0, 0).contains(0));
    assertTrue(sp.getForwardingTargetFields(1, 0).contains(1));
    assertTrue(sp.getForwardingTargetFields(1, 1).contains(2));
    assertTrue(sp.getForwardingTargetFields(1, 2).contains(3));
    assertTrue(sp.getForwardingTargetFields(1, 3).contains(4));
    assertTrue(sp.getForwardingTargetFields(1, 4).contains(5));
    assertTrue(sp.getForwardingTargetFields(1, 5).contains(6));
}