Examples with FieldSet org.apache.flink.api.common.operators.util.FieldSet used on opensource projects

Example 26 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class SemanticPropertiesTranslationTest method testUnaryFunctionForwardedInLine2.

@Test
public void testUnaryFunctionForwardedInLine2() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
    input.map(new ReadSetMapper<Tuple3<Long, Long, Long>>()).withForwardedFields("0->1; 2").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertTrue(fw1.contains(1));
    assertTrue(fw2.contains(2));
}

Example 27 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class SemanticPropertiesPrecedenceTest method testFunctionApiPrecedence.

@Test
public void testFunctionApiPrecedence() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(Tuple3.of(3L, "test", 42));
    input.map(new WildcardForwardedMapper<Tuple3<Long, String, Integer>>()).withForwardedFields("f0").output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
    FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertNotNull(fw3);
    assertTrue(fw1.contains(0));
    assertFalse(fw2.contains(1));
    assertFalse(fw3.contains(2));
}

Example 28 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class GlobalProperties method filterBySemanticProperties.

/**
 * Filters these GlobalProperties by the fields that are forwarded to the output as described by
 * the SemanticProperties.
 *
 * @param props The semantic properties holding information about forwarded fields.
 * @param input The index of the input.
 * @return The filtered GlobalProperties
 */
public GlobalProperties filterBySemanticProperties(SemanticProperties props, int input) {
    if (props == null) {
        throw new NullPointerException("SemanticProperties may not be null.");
    }
    GlobalProperties gp = new GlobalProperties();
    // filter partitioning
    switch(this.partitioning) {
        case RANGE_PARTITIONED:
            // check if ordering is preserved
            Ordering newOrdering = new Ordering();
            for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
                int sourceField = this.ordering.getInvolvedIndexes().get(i);
                FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
                if (targetField == null || targetField.size() == 0) {
                    // partitioning is destroyed
                    newOrdering = null;
                    break;
                } else {
                    // field equivalence sets in the future.
                    if (targetField.size() > 1) {
                        LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                    }
                    newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
                }
            }
            if (newOrdering != null) {
                gp.partitioning = PartitioningProperty.RANGE_PARTITIONED;
                gp.ordering = newOrdering;
                gp.partitioningFields = newOrdering.getInvolvedIndexes();
                gp.distribution = this.distribution;
            }
            break;
        case HASH_PARTITIONED:
        case ANY_PARTITIONING:
        case CUSTOM_PARTITIONING:
            FieldList newPartitioningFields = new FieldList();
            for (int sourceField : this.partitioningFields) {
                FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
                if (targetField == null || targetField.size() == 0) {
                    newPartitioningFields = null;
                    break;
                } else {
                    // field equivalence sets in the future.
                    if (targetField.size() > 1) {
                        LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                    }
                    newPartitioningFields = newPartitioningFields.addField(targetField.toArray()[0]);
                }
            }
            if (newPartitioningFields != null) {
                gp.partitioning = this.partitioning;
                gp.partitioningFields = newPartitioningFields;
                gp.customPartitioner = this.customPartitioner;
            }
            break;
        case FORCED_REBALANCED:
        case FULL_REPLICATION:
        case RANDOM_PARTITIONED:
            gp.partitioning = this.partitioning;
            break;
        default:
            throw new RuntimeException("Unknown partitioning type.");
    }
    // filter unique field combinations
    if (this.uniqueFieldCombinations != null) {
        Set<FieldSet> newUniqueFieldCombinations = new HashSet<FieldSet>();
        for (FieldSet fieldCombo : this.uniqueFieldCombinations) {
            FieldSet newFieldCombo = new FieldSet();
            for (Integer sourceField : fieldCombo) {
                FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
                if (targetField == null || targetField.size() == 0) {
                    newFieldCombo = null;
                    break;
                } else {
                    // field equivalence sets in the future.
                    if (targetField.size() > 1) {
                        LOG.warn("Found that a field is forwarded to more than one target field in " + "semantic forwarded field information. Will only use the field with the lowest index.");
                    }
                    newFieldCombo = newFieldCombo.addField(targetField.toArray()[0]);
                }
            }
            if (newFieldCombo != null) {
                newUniqueFieldCombinations.add(newFieldCombo);
            }
        }
        if (!newUniqueFieldCombinations.isEmpty()) {
            gp.uniqueFieldCombinations = newUniqueFieldCombinations;
        }
    }
    return gp;
}

Example 29 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class CoGroupNode method getSemanticPropertiesForLocalPropertyFiltering.

@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
    // Local properties for CoGroup may only be preserved on key fields.
    DualInputSemanticProperties origProps = ((DualInputOperator<?, ?, ?, ?>) getOperator()).getSemanticProperties();
    DualInputSemanticProperties filteredProps = new DualInputSemanticProperties();
    FieldSet readSet1 = origProps.getReadFields(0);
    FieldSet readSet2 = origProps.getReadFields(1);
    if (readSet1 != null) {
        filteredProps.addReadFields(0, readSet1);
    }
    if (readSet2 != null) {
        filteredProps.addReadFields(1, readSet2);
    }
    // preserve only key fields (first input)
    for (int f : this.keys1) {
        FieldSet targets = origProps.getForwardingTargetFields(0, f);
        for (int t : targets) {
            filteredProps.addForwardedField(0, f, t);
        }
    }
    // preserve only key fields (second input)
    for (int f : this.keys2) {
        FieldSet targets = origProps.getForwardingTargetFields(1, f);
        for (int t : targets) {
            filteredProps.addForwardedField(1, f, t);
        }
    }
    return filteredProps;
}

Example 30 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class DualInputSemanticPropertiesTest method testGetReadSet.

@Test
public void testGetReadSet() {
    // first input
    DualInputSemanticProperties sp = new DualInputSemanticProperties();
    sp.addReadFields(0, new FieldSet(0, 1));
    assertEquals(2, sp.getReadFields(0).size());
    assertTrue(sp.getReadFields(0).contains(0));
    assertTrue(sp.getReadFields(0).contains(1));
    sp.addReadFields(0, new FieldSet(3));
    assertEquals(3, sp.getReadFields(0).size());
    assertTrue(sp.getReadFields(0).contains(0));
    assertTrue(sp.getReadFields(0).contains(1));
    assertTrue(sp.getReadFields(0).contains(3));
    // second input
    sp = new DualInputSemanticProperties();
    sp.addReadFields(1, new FieldSet(0, 1));
    assertEquals(2, sp.getReadFields(1).size());
    assertTrue(sp.getReadFields(1).contains(0));
    assertTrue(sp.getReadFields(1).contains(1));
    sp.addReadFields(1, new FieldSet(3));
    assertEquals(3, sp.getReadFields(1).size());
    assertTrue(sp.getReadFields(1).contains(0));
    assertTrue(sp.getReadFields(1).contains(1));
    assertTrue(sp.getReadFields(1).contains(3));
}