Examples with SingleInputSemanticProperties org.apache.flink.api.common.operators.SingleInputSemanticProperties used on opensource projects

Example 56 with SingleInputSemanticProperties

use of org.apache.flink.api.common.operators.SingleInputSemanticProperties in project flink by apache.

the class RequestedGlobalPropertiesFilteringTest method testAnyPartitioningPreserved1.

@Test
public void testAnyPartitioningPreserved1() {
    SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[] { "0;3;4" }, null, null, tupleInfo, tupleInfo);
    RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
    rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
    RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
    assertNotNull(filtered);
    assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
    assertNotNull(filtered.getPartitionedFields());
    assertEquals(3, filtered.getPartitionedFields().size());
    assertTrue(filtered.getPartitionedFields().contains(0));
    assertTrue(filtered.getPartitionedFields().contains(3));
    assertTrue(filtered.getPartitionedFields().contains(4));
    assertNull(filtered.getDataDistribution());
    assertNull(filtered.getCustomPartitioner());
    assertNull(filtered.getOrdering());
}

Example 57 with SingleInputSemanticProperties

use of org.apache.flink.api.common.operators.SingleInputSemanticProperties in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testGroupingPreserved2.

@Test
public void testGroupingPreserved2() {
    SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[] { "3->0;5->2;1->3" }, null, null, tupleInfo, tupleInfo);
    RequestedLocalProperties rlProp = new RequestedLocalProperties();
    rlProp.setGroupedFields(new FieldSet(0, 2, 3));
    RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
    assertNotNull(filtered);
    assertNotNull(filtered.getGroupedFields());
    assertEquals(3, filtered.getGroupedFields().size());
    assertTrue(filtered.getGroupedFields().contains(3));
    assertTrue(filtered.getGroupedFields().contains(5));
    assertTrue(filtered.getGroupedFields().contains(1));
    assertNull(filtered.getOrdering());
}

Example 58 with SingleInputSemanticProperties

use of org.apache.flink.api.common.operators.SingleInputSemanticProperties in project flink by apache.

the class RequestedLocalPropertiesFilteringTest method testGroupingErased.

@Test
public void testGroupingErased() {
    SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[] { "0;2" }, null, null, tupleInfo, tupleInfo);
    RequestedLocalProperties rlProp = new RequestedLocalProperties();
    rlProp.setGroupedFields(new FieldSet(0, 2, 3));
    RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
    assertNull(filtered);
}

Example 59 with SingleInputSemanticProperties

use of org.apache.flink.api.common.operators.SingleInputSemanticProperties in project flink by apache.

the class ScalaAggregateOperator method translateToDataFlow.

@SuppressWarnings("unchecked")
@Override
protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> translateToDataFlow(Operator<IN> input) {
    // sanity check
    if (this.aggregationFunctions.isEmpty() || this.aggregationFunctions.size() != this.fields.size()) {
        throw new IllegalStateException();
    }
    // construct the aggregation function
    AggregationFunction<Object>[] aggFunctions = new AggregationFunction[this.aggregationFunctions.size()];
    int[] fields = new int[this.fields.size()];
    StringBuilder genName = new StringBuilder();
    for (int i = 0; i < fields.length; i++) {
        aggFunctions[i] = (AggregationFunction<Object>) this.aggregationFunctions.get(i);
        fields[i] = this.fields.get(i);
        genName.append(aggFunctions[i].toString()).append('(').append(fields[i]).append(')').append(',');
    }
    genName.setLength(genName.length() - 1);
    @SuppressWarnings("rawtypes") RichGroupReduceFunction<IN, IN> function = new AggregatingUdf(getInputType(), aggFunctions, fields);
    String name = getName() != null ? getName() : genName.toString();
    // distinguish between grouped reduce and non-grouped reduce
    if (this.grouping == null) {
        // non grouped aggregation
        UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType());
        GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po = new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, new int[0], name);
        po.setCombinable(true);
        // set input
        po.setInput(input);
        // set parallelism
        po.setParallelism(this.getParallelism());
        return po;
    }
    if (this.grouping.getKeys() instanceof Keys.ExpressionKeys) {
        // grouped aggregation
        int[] logicalKeyPositions = this.grouping.getKeys().computeLogicalKeyPositions();
        UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<>(getInputType(), getResultType());
        GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po = new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, logicalKeyPositions, name);
        po.setCombinable(true);
        // set input
        po.setInput(input);
        // set parallelism
        po.setParallelism(this.getParallelism());
        SingleInputSemanticProperties props = new SingleInputSemanticProperties();
        for (int keyField : logicalKeyPositions) {
            boolean keyFieldUsedInAgg = false;
            for (int aggField : fields) {
                if (keyField == aggField) {
                    keyFieldUsedInAgg = true;
                    break;
                }
            }
            if (!keyFieldUsedInAgg) {
                props.addForwardedField(keyField, keyField);
            }
        }
        po.setSemanticProperties(props);
        po.setCustomPartitioner(grouping.getCustomPartitioner());
        return po;
    } else if (this.grouping.getKeys() instanceof Keys.SelectorFunctionKeys) {
        throw new UnsupportedOperationException("Aggregate does not support grouping with KeySelector functions, yet.");
    } else {
        throw new UnsupportedOperationException("Unrecognized key type.");
    }
}

Example 60 with SingleInputSemanticProperties

use of org.apache.flink.api.common.operators.SingleInputSemanticProperties in project flink by apache.

the class SemanticPropUtilTest method testNonForwardedInvalidTypes5.

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedInvalidTypes5() {
    String[] nonForwardedFields = { "int1" };
    SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, null, pojoType, pojo2Type);
}