use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.
the class ReduceOperatorTest method testSemanticPropsWithKeySelector2.
@Test
public void testSemanticPropsWithKeySelector2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduce(new DummyReduceFunction2()).withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.
the class GenericFlatTypePostPass method addSchemaToSchema.
private void addSchemaToSchema(T sourceSchema, T targetSchema, OptimizerNode optNode, int input) {
try {
for (Map.Entry<Integer, X> entry : sourceSchema) {
Integer pos = entry.getKey();
SemanticProperties sprops = optNode.getSemanticProperties();
if (sprops != null && sprops.getForwardingTargetFields(input, pos) != null && sprops.getForwardingTargetFields(input, pos).contains(pos)) {
targetSchema.addType(pos, entry.getValue());
}
}
} catch (ConflictingFieldTypeInfoException e) {
throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber() + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " + "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
}
}
use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.
the class InterestingProperties method filterByCodeAnnotations.
public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input) {
InterestingProperties iProps = new InterestingProperties();
SemanticProperties props;
if (node instanceof SingleInputNode || node instanceof TwoInputNode) {
props = node.getSemanticProperties();
} else {
props = new SemanticProperties.EmptySemanticProperties();
}
for (RequestedGlobalProperties rgp : this.globalProps) {
RequestedGlobalProperties filtered = rgp.filterBySemanticProperties(props, input);
if (filtered != null && !filtered.isTrivial()) {
iProps.addGlobalProperties(filtered);
}
}
for (RequestedLocalProperties rlp : this.localProps) {
RequestedLocalProperties filtered = rlp.filterBySemanticProperties(props, input);
if (filtered != null && !filtered.isTrivial()) {
iProps.addLocalProperties(filtered);
}
}
return iProps;
}
use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.
the class UdfAnalyzer method getSemanticProperties.
public SemanticProperties getSemanticProperties() {
final SemanticProperties sp;
if (isBinary) {
sp = new DualInputSemanticProperties();
if (returnValue != null) {
String[] ff1Array = null;
final String ff1 = returnValue.toForwardedFieldsExpression(Input.INPUT_1);
if (ff1 != null && ff1.length() > 0) {
ff1Array = new String[] { ff1 };
}
String[] ff2Array = null;
final String ff2 = returnValue.toForwardedFieldsExpression(Input.INPUT_2);
if (ff2 != null && ff2.length() > 0) {
ff2Array = new String[] { ff2 };
}
SemanticPropUtil.getSemanticPropsDualFromString((DualInputSemanticProperties) sp, ff1Array, ff2Array, null, null, null, null, in1Type, in2Type, outType, true);
}
} else {
sp = new SingleInputSemanticProperties();
if (returnValue != null) {
String[] ffArray = null;
final String ff = returnValue.toForwardedFieldsExpression(Input.INPUT_1);
if (ff != null && ff.length() > 0) {
ffArray = new String[] { ff };
}
SemanticPropUtil.getSemanticPropsSingleFromString((SingleInputSemanticProperties) sp, ffArray, null, null, in1Type, outType, true);
}
}
return sp;
}
use of org.apache.flink.api.common.operators.SemanticProperties in project flink by apache.
the class SemanticPropUtilTest method testAddSourceFieldOffset.
// --------------------------------------------------------------------------------------------
// Offset
// --------------------------------------------------------------------------------------------
@Test
public void testAddSourceFieldOffset() {
SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
semProps.addForwardedField(0, 1);
semProps.addForwardedField(0, 4);
semProps.addForwardedField(2, 0);
semProps.addForwardedField(4, 3);
semProps.addReadFields(new FieldSet(0, 3));
SemanticProperties offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 5, 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 2);
assertTrue(offsetProps.getForwardingTargetFields(0, 0).contains(1));
assertTrue(offsetProps.getForwardingTargetFields(0, 0).contains(4));
assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 2).contains(0));
assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 4).contains(3));
assertTrue(offsetProps.getReadFields(0).size() == 2);
assertTrue(offsetProps.getReadFields(0).contains(0));
assertTrue(offsetProps.getReadFields(0).contains(3));
offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 5, 3);
assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(4));
assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 5).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 5).contains(0));
assertTrue(offsetProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 7).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 7).contains(3));
assertTrue(offsetProps.getReadFields(0).size() == 2);
assertTrue(offsetProps.getReadFields(0).contains(3));
assertTrue(offsetProps.getReadFields(0).contains(6));
semProps = new SingleInputSemanticProperties();
SemanticPropUtil.addSourceFieldOffset(semProps, 1, 0);
semProps = new SingleInputSemanticProperties();
semProps.addForwardedField(0, 0);
semProps.addForwardedField(1, 2);
semProps.addForwardedField(2, 4);
offsetProps = SemanticPropUtil.addSourceFieldOffset(semProps, 3, 2);
assertTrue(offsetProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(offsetProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 2).contains(0));
assertTrue(offsetProps.getForwardingTargetFields(0, 3).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 3).contains(2));
assertTrue(offsetProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(offsetProps.getForwardingTargetFields(0, 4).contains(4));
}
Aggregations