Examples with FieldSet org.apache.flink.api.common.operators.util.FieldSet used on opensource projects

Example 56 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class GroupReduceNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
    origProps.addForwardedField(0, 1);
    origProps.addForwardedField(2, 2);
    origProps.addForwardedField(3, 4);
    origProps.addForwardedField(6, 0);
    origProps.addReadFields(new FieldSet(0, 2, 4, 7));
    GroupReduceOperatorBase<?, ?, ?> op = mock(GroupReduceOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getParameters()).thenReturn(new Configuration());
    GroupReduceNode node = new GroupReduceNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
}

Example 57 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class GroupCombineNode method getSemanticPropertiesForLocalPropertyFiltering.

@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
    // Local properties for GroupCombine may only be preserved on key fields.
    SingleInputSemanticProperties origProps = ((SingleInputOperator<?, ?, ?>) getOperator()).getSemanticProperties();
    SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
    FieldSet readSet = origProps.getReadFields(0);
    if (readSet != null) {
        filteredProps.addReadFields(readSet);
    }
    // only add forward field information for key fields
    if (this.keys != null) {
        for (int f : this.keys) {
            FieldSet targets = origProps.getForwardingTargetFields(0, f);
            for (int t : targets) {
                filteredProps.addForwardedField(f, t);
            }
        }
    }
    return filteredProps;
}

Example 58 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class GroupReduceNode method getSemanticPropertiesForLocalPropertyFiltering.

@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
    // Local properties for GroupReduce may only be preserved on key fields.
    SingleInputSemanticProperties origProps = getOperator().getSemanticProperties();
    SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
    FieldSet readSet = origProps.getReadFields(0);
    if (readSet != null) {
        filteredProps.addReadFields(readSet);
    }
    // only add forward field information for key fields
    if (this.keys != null) {
        for (int f : this.keys) {
            FieldSet targets = origProps.getForwardingTargetFields(0, f);
            for (int t : targets) {
                filteredProps.addForwardedField(f, t);
            }
        }
    }
    return filteredProps;
}

Example 59 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class MapPartitionNode method getSemanticPropertiesForLocalPropertyFiltering.

@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
    // Local properties for MapPartition may not be preserved.
    SingleInputSemanticProperties origProps = ((SingleInputOperator<?, ?, ?>) getOperator()).getSemanticProperties();
    SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
    FieldSet readSet = origProps.getReadFields(0);
    if (readSet != null) {
        filteredProps.addReadFields(readSet);
    }
    return filteredProps;
}

Example 60 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class LocalProperties method filterBySemanticProperties.

// --------------------------------------------------------------------------------------------
/**
	 * Filters these LocalProperties by the fields that are forwarded to the output
	 * as described by the SemanticProperties.
	 *
	 * @param props The semantic properties holding information about forwarded fields.
	 * @param input The index of the input.
	 * @return The filtered LocalProperties
	 */
public LocalProperties filterBySemanticProperties(SemanticProperties props, int input) {
    if (props == null) {
        throw new NullPointerException("SemanticProperties may not be null.");
    }
    LocalProperties returnProps = new LocalProperties();
    // check if sorting is preserved
    if (this.ordering != null) {
        Ordering newOrdering = new Ordering();
        for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
            int sourceField = this.ordering.getInvolvedIndexes().get(i);
            FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
            if (targetField == null || targetField.size() == 0) {
                if (i == 0) {
                    // order fully destroyed
                    newOrdering = null;
                    break;
                } else {
                    // order partially preserved
                    break;
                }
            } else {
                // use any field of target fields for now.  We should use something like field equivalence sets in the future.
                if (targetField.size() > 1) {
                    LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                }
                newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
            }
        }
        returnProps.ordering = newOrdering;
        if (newOrdering != null) {
            returnProps.groupedFields = newOrdering.getInvolvedIndexes();
        } else {
            returnProps.groupedFields = null;
        }
    } else // check if grouping is preserved
    if (this.groupedFields != null) {
        FieldList newGroupedFields = new FieldList();
        for (Integer sourceField : this.groupedFields) {
            FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
            if (targetField == null || targetField.size() == 0) {
                newGroupedFields = null;
                break;
            } else {
                // use any field of target fields for now.  We should use something like field equivalence sets in the future.
                if (targetField.size() > 1) {
                    LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                }
                newGroupedFields = newGroupedFields.addField(targetField.toArray()[0]);
            }
        }
        returnProps.groupedFields = newGroupedFields;
    }
    if (this.uniqueFields != null) {
        Set<FieldSet> newUniqueFields = new HashSet<FieldSet>();
        for (FieldSet fields : this.uniqueFields) {
            FieldSet newFields = new FieldSet();
            for (Integer sourceField : fields) {
                FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
                if (targetField == null || targetField.size() == 0) {
                    newFields = null;
                    break;
                } else {
                    // use any field of target fields for now.  We should use something like field equivalence sets in the future.
                    if (targetField.size() > 1) {
                        LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                    }
                    newFields = newFields.addField(targetField.toArray()[0]);
                }
            }
            if (newFields != null) {
                newUniqueFields.add(newFields);
            }
        }
        if (!newUniqueFields.isEmpty()) {
            returnProps.uniqueFields = newUniqueFields;
        } else {
            returnProps.uniqueFields = null;
        }
    }
    return returnProps;
}