Examples with FieldSet org.apache.flink.api.common.operators.util.FieldSet used on opensource projects

Example 51 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testDualGroupingPreserved.

@Test
public void testDualGroupingPreserved() {
    DualInputSemanticProperties dprops = new DualInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[] { "1->0;3;2->4" }, new String[] { "0->7;1" }, null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
    RequestedLocalProperties lprops1 = new RequestedLocalProperties();
    lprops1.setGroupedFields(new FieldSet(0, 3, 4));
    RequestedLocalProperties lprops2 = new RequestedLocalProperties();
    lprops2.setGroupedFields(new FieldSet(7, 1));
    RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
    RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);
    assertNotNull(filtered1);
    assertNotNull(filtered1.getGroupedFields());
    assertEquals(3, filtered1.getGroupedFields().size());
    assertTrue(filtered1.getGroupedFields().contains(1));
    assertTrue(filtered1.getGroupedFields().contains(2));
    assertTrue(filtered1.getGroupedFields().contains(3));
    assertNull(filtered1.getOrdering());
    assertNotNull(filtered2);
    assertNotNull(filtered2.getGroupedFields());
    assertEquals(2, filtered2.getGroupedFields().size());
    assertTrue(filtered2.getGroupedFields().contains(0));
    assertTrue(filtered2.getGroupedFields().contains(1));
    assertNull(filtered2.getOrdering());
}

Example 52 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testGroupingPreserved1.

@Test
public void testGroupingPreserved1() {
    SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[] { "0;2;3" }, null, null, tupleInfo, tupleInfo);
    RequestedLocalProperties rlProp = new RequestedLocalProperties();
    rlProp.setGroupedFields(new FieldSet(0, 2, 3));
    RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
    assertNotNull(filtered);
    assertNotNull(filtered.getGroupedFields());
    assertEquals(3, filtered.getGroupedFields().size());
    assertTrue(filtered.getGroupedFields().contains(0));
    assertTrue(filtered.getGroupedFields().contains(2));
    assertTrue(filtered.getGroupedFields().contains(3));
    assertNull(filtered.getOrdering());
}

Example 53 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testInvalidInputIndex.

@Test(expected = IndexOutOfBoundsException.class)
public void testInvalidInputIndex() {
    SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[] { "1; 4" }, null, null, tupleInfo, tupleInfo);
    RequestedLocalProperties rlProp = new RequestedLocalProperties();
    rlProp.setGroupedFields(new FieldSet(1, 4));
    rlProp.filterBySemanticProperties(sProps, 1);
}

Example 54 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class CoGroupNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    DualInputSemanticProperties origProps = new DualInputSemanticProperties();
    // props for first input
    origProps.addForwardedField(0, 0, 1);
    origProps.addForwardedField(0, 2, 2);
    origProps.addForwardedField(0, 3, 4);
    origProps.addForwardedField(0, 6, 0);
    origProps.addReadFields(0, new FieldSet(0, 2, 4, 7));
    // props for second input
    origProps.addForwardedField(1, 1, 2);
    origProps.addForwardedField(1, 2, 8);
    origProps.addForwardedField(1, 3, 7);
    origProps.addForwardedField(1, 6, 6);
    origProps.addReadFields(1, new FieldSet(1, 3, 4));
    CoGroupOperatorBase<?, ?, ?, ?> op = mock(CoGroupOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getKeyColumns(1)).thenReturn(new int[] { 6, 3 });
    when(op.getParameters()).thenReturn(new Configuration());
    CoGroupNode node = new CoGroupNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    // check first input props
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    // check second input props
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 1).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).contains(6));
    assertTrue(filteredProps.getForwardingSourceField(1, 2) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 8) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 7) == 3);
    assertTrue(filteredProps.getForwardingSourceField(1, 6) == 6);
    assertTrue(filteredProps.getReadFields(1).size() == 3);
    assertTrue(filteredProps.getReadFields(1).contains(1));
    assertTrue(filteredProps.getReadFields(1).contains(3));
    assertTrue(filteredProps.getReadFields(1).contains(4));
}

Example 55 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class GroupCombineNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
    origProps.addForwardedField(0, 1);
    origProps.addForwardedField(2, 2);
    origProps.addForwardedField(3, 4);
    origProps.addForwardedField(6, 0);
    origProps.addReadFields(new FieldSet(0, 2, 4, 7));
    GroupCombineOperatorBase<?, ?, ?> op = mock(GroupCombineOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getParameters()).thenReturn(new Configuration());
    GroupCombineNode node = new GroupCombineNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
}