Examples with SemanticProperties org.apache.flink.api.common.operators.SemanticProperties used on opensource projects

Example 16 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class ReduceOperatorTest method testSemanticPropsWithKeySelector2.

@Test
public void testSemanticPropsWithKeySelector2() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
    ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduce(new DummyReduceFunction2()).withForwardedFields("0->4;1;1->3;2");
    SemanticProperties semProps = reduceOp.getSemanticProperties();
    assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
    assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
    assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
    assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
    assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
    assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
    assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
    assertTrue(semProps.getReadFields(0).size() == 3);
    assertTrue(semProps.getReadFields(0).contains(2));
    assertTrue(semProps.getReadFields(0).contains(5));
    assertTrue(semProps.getReadFields(0).contains(6));
}

Example 17 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class GenericFlatTypePostPass method addSchemaToSchema.

private void addSchemaToSchema(T sourceSchema, T targetSchema, OptimizerNode optNode, int input) {
    try {
        for (Map.Entry<Integer, X> entry : sourceSchema) {
            Integer pos = entry.getKey();
            SemanticProperties sprops = optNode.getSemanticProperties();
            if (sprops != null && sprops.getForwardingTargetFields(input, pos) != null && sprops.getForwardingTargetFields(input, pos).contains(pos)) {
                targetSchema.addType(pos, entry.getValue());
            }
        }
    } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber() + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " + "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
    }
}

Example 18 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class InterestingProperties method filterByCodeAnnotations.

public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input) {
    InterestingProperties iProps = new InterestingProperties();
    SemanticProperties props;
    if (node instanceof SingleInputNode || node instanceof TwoInputNode) {
        props = node.getSemanticProperties();
    } else {
        props = new SemanticProperties.EmptySemanticProperties();
    }
    for (RequestedGlobalProperties rgp : this.globalProps) {
        RequestedGlobalProperties filtered = rgp.filterBySemanticProperties(props, input);
        if (filtered != null && !filtered.isTrivial()) {
            iProps.addGlobalProperties(filtered);
        }
    }
    for (RequestedLocalProperties rlp : this.localProps) {
        RequestedLocalProperties filtered = rlp.filterBySemanticProperties(props, input);
        if (filtered != null && !filtered.isTrivial()) {
            iProps.addLocalProperties(filtered);
        }
    }
    return iProps;
}

Example 19 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class UdfAnalyzer method getSemanticProperties.

public SemanticProperties getSemanticProperties() {
    final SemanticProperties sp;
    if (isBinary) {
        sp = new DualInputSemanticProperties();
        if (returnValue != null) {
            String[] ff1Array = null;
            final String ff1 = returnValue.toForwardedFieldsExpression(Input.INPUT_1);
            if (ff1 != null && ff1.length() > 0) {
                ff1Array = new String[] { ff1 };
            }
            String[] ff2Array = null;
            final String ff2 = returnValue.toForwardedFieldsExpression(Input.INPUT_2);
            if (ff2 != null && ff2.length() > 0) {
                ff2Array = new String[] { ff2 };
            }
            SemanticPropUtil.getSemanticPropsDualFromString((DualInputSemanticProperties) sp, ff1Array, ff2Array, null, null, null, null, in1Type, in2Type, outType, true);
        }
    } else {
        sp = new SingleInputSemanticProperties();
        if (returnValue != null) {
            String[] ffArray = null;
            final String ff = returnValue.toForwardedFieldsExpression(Input.INPUT_1);
            if (ff != null && ff.length() > 0) {
                ffArray = new String[] { ff };
            }
            SemanticPropUtil.getSemanticPropsSingleFromString((SingleInputSemanticProperties) sp, ffArray, null, null, in1Type, outType, true);
        }
    }
    return sp;
}

Example 20 with SemanticProperties

use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.

the class SemanticPropUtilTest method testAddSourceFieldOffset.

// --------------------------------------------------------------------------------------------
// Offset
// --------------------------------------------------------------------------------------------
@Test
public void testAddSourceFieldOffset() {
    SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
    semProps.addForwardedField(0, 1);
    semProps.addForwardedField(0, 4);
    semProps.addForwardedField(2, 0);
    semProps.addForwardedField(4, 3);
    semProps.addReadFields(new FieldSet(0, 3));
    SemanticProperties offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 5, 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 2);
    assertTrue(offsetProps.getForwardingTargetFields(0, 0).contains(1));
    assertTrue(offsetProps.getForwardingTargetFields(0, 0).contains(4));
    assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 2).contains(0));
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 4).contains(3));
    assertTrue(offsetProps.getReadFields(0).size() == 2);
    assertTrue(offsetProps.getReadFields(0).contains(0));
    assertTrue(offsetProps.getReadFields(0).contains(3));
    offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 5, 3);
    assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 2);
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(1));
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 5).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 5).contains(0));
    assertTrue(offsetProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 7).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 7).contains(3));
    assertTrue(offsetProps.getReadFields(0).size() == 2);
    assertTrue(offsetProps.getReadFields(0).contains(3));
    assertTrue(offsetProps.getReadFields(0).contains(6));
    semProps = new SingleInputSemanticProperties();
    SemanticPropUtil.addSourceFieldOffset(semProps, 1, 0);
    semProps = new SingleInputSemanticProperties();
    semProps.addForwardedField(0, 0);
    semProps.addForwardedField(1, 2);
    semProps.addForwardedField(2, 4);
    offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 3, 2);
    assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
    assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 2).contains(0));
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(2));
    assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 1);
    assertTrue(offsetProps.getForwardingTargetFields(0, 4).contains(4));
}