Examples with FieldSet org.apache.flink.api.common.operators.util.FieldSet used on opensource projects

Example 21 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class RequestedGlobalPropertiesFilteringTest method testAnyPartitioningErased.

@Test
public void testAnyPartitioningErased() {
    SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[] { "1;2" }, null, null, tupleInfo, tupleInfo);
    RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
    rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
    RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
    assertNull(filtered);
}

Example 22 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class FeedbackPropertiesMatchTest method testSingleInputOperatorsChainOfThree.

@Test
public void testSingleInputOperatorsChainOfThree() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toMap3 = new Channel(map2);
        SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP);
        // set local strategy in first channel, so later non matching local properties do not
        // matter
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap3.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            toMap3.setRequiredGlobalProps(null);
            toMap3.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // set global strategy in first channel, so later non matching global properties do not
        // matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(5, 3));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 3));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(1, 2), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap3.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            toMap3.setRequiredGlobalProps(reqGp);
            toMap3.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 23 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class PlanNode method updatePropertiesWithUniqueSets.

// --------------------------------------------------------------------------------------------
// Miscellaneous
// --------------------------------------------------------------------------------------------
public void updatePropertiesWithUniqueSets(Set<FieldSet> uniqueFieldCombinations) {
    if (uniqueFieldCombinations == null || uniqueFieldCombinations.isEmpty()) {
        return;
    }
    for (FieldSet fields : uniqueFieldCombinations) {
        this.globalProps.addUniqueFieldCombination(fields);
        this.localProps = this.localProps.addUniqueFields(fields);
    }
}

Example 24 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class SemanticPropertiesTranslationTest method testUnaryFunctionWildcardForwardedAnnotation.

@Test
public void testUnaryFunctionWildcardForwardedAnnotation() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(new Tuple3<Long, String, Integer>(3L, "test", 42));
    input.map(new WildcardForwardedMapper<Tuple3<Long, String, Integer>>()).output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
    FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertNotNull(fw3);
    assertTrue(fw1.contains(0));
    assertTrue(fw2.contains(1));
    assertTrue(fw3.contains(2));
}

Example 25 with FieldSet

use of org.apache.flink.api.common.operators.util.FieldSet in project flink by apache.

the class SemanticPropertiesTranslationTest method testUnaryFunctionAllForwardedExceptAnnotation.

@Test
public void testUnaryFunctionAllForwardedExceptAnnotation() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
    input.map(new AllForwardedExceptMapper<Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertTrue(fw1.contains(0));
    assertTrue(fw2.contains(2));
}