Examples with Channel org.apache.flink.optimizer.plan.Channel used on opensource projects

Example 46 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class FeedbackPropertiesMatchTest method testSingleInputOperators.

@Test
public void testSingleInputOperators() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        // no feedback properties and none are ever required and present
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = new LocalProperties();
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some global feedback properties and none are ever required and present
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = new LocalProperties();
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some local feedback properties and none are ever required and present
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some global and local feedback properties and none are ever required and present
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // --------------------------- requirements on channel 1 -----------------------
        // some required global properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = new LocalProperties();
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldList(2, 5));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required local properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1, 2));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global and local properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldList(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1, 2));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global and local properties, which are over-fulfilled
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldSet(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global properties that are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 1));
            LocalProperties lp = new LocalProperties();
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldList(2, 5));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required local properties that are not met
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required global and local properties where the global properties are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 1));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required global and local properties where the local properties are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // --------------------------- requirements on channel 2 -----------------------
        // some required global properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = new LocalProperties();
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldList(2, 5));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required local properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1, 2));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global and local properties, which are matched exactly
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 5));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldList(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1, 2));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global and local properties, which are over-fulfilled
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldSet(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global properties that are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 1));
            LocalProperties lp = new LocalProperties();
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setHashPartitioned(new FieldSet(2, 5));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required local properties that are not met
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required global and local properties where the global properties are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(2, 1));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 5));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some required global and local properties where the local properties are not met
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // ---------------------- requirements mixed on 1 and 2 -----------------------
        // some required global properties at step one and some more at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp1 = new RequestedGlobalProperties();
            reqGp1.setAnyPartitioning(new FieldList(1, 2));
            RequestedGlobalProperties reqGp2 = new RequestedGlobalProperties();
            reqGp2.setHashPartitioned(new FieldList(1, 2));
            toMap1.setRequiredGlobalProps(reqGp1);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp2);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required local properties at step one and some more at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp1 = new RequestedLocalProperties();
            reqLp1.setGroupedFields(new FieldList(3, 1));
            RequestedLocalProperties reqLp2 = new RequestedLocalProperties();
            reqLp2.setOrdering(new Ordering(3, null, Order.ANY).appendOrdering(1, null, Order.ANY));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp1);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp2);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required global properties at step one and some local ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some required local properties at step one and some global ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // some fulfilled global properties at step one and some non-fulfilled local ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 3));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some fulfilled local properties at step one and some non-fulfilled global ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(2, 3));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some non-fulfilled global properties at step one and some fulfilled local ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(2, 3));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 1));
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // some non-fulfilled local properties at step one and some fulfilled global ones at step 2
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldList(1, 2));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(2, 1, 3));
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 47 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class FeedbackPropertiesMatchTest method testSingleInputOperatorsWithReCreation.

@Test
public void testSingleInputOperatorsWithReCreation() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
        Channel toMap1 = new Channel(target);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(map1);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        // set ship strategy in first channel, so later non matching global properties do not matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 5));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(reqGp);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
        // set ship strategy in second channel, so previous non matching global properties void the match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.EMPTY;
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(2, 5));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // set local strategy in first channel, so later non matching local properties do not matter
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(null);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(reqLp);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
        }
        // set local strategy in second channel, so previous non matching local properties void the match
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(4, 1));
            toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.NONE);
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap1.setRequiredGlobalProps(null);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(NOT_MET, report);
        }
        // create the properties on the same node as the requirement
        {
            GlobalProperties gp = new GlobalProperties();
            gp.setHashPartitioned(new FieldList(1, 2));
            LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
            RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
            reqGp.setAnyPartitioning(new FieldSet(5, 7));
            RequestedLocalProperties reqLp = new RequestedLocalProperties();
            reqLp.setGroupedFields(new FieldList(5, 7));
            toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(5, 7), DataExchangeMode.PIPELINED);
            toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] { false, false });
            toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
            toMap2.setLocalStrategy(LocalStrategy.NONE);
            toMap1.setRequiredGlobalProps(reqGp);
            toMap1.setRequiredLocalProps(reqLp);
            toMap2.setRequiredGlobalProps(null);
            toMap2.setRequiredLocalProps(null);
            FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
            assertEquals(MET, report);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 48 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class FeedbackPropertiesMatchTest method testNoPartialSolutionFoundTwoInputOperator.

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
    try {
        SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
        SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
        SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
        Channel toMap1 = new Channel(source1);
        toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap1.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
        Channel toMap2 = new Channel(source2);
        toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toMap2.setLocalStrategy(LocalStrategy.NONE);
        SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
        Channel toJoin1 = new Channel(map1);
        Channel toJoin2 = new Channel(map2);
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
        assertEquals(NO_PARTIAL_SOLUTION, report);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 49 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class GroupOrderTest method testCoGroupWithGroupOrder.

@Test
public void testCoGroupWithGroupOrder() {
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Tuple7<Long, Long, Long, Long, Long, Long, Long>> set1 = env.readCsvFile("/tmp/fake1.csv").types(Long.class, Long.class, Long.class, Long.class, Long.class, Long.class, Long.class);
    DataSet<Tuple7<Long, Long, Long, Long, Long, Long, Long>> set2 = env.readCsvFile("/tmp/fake2.csv").types(Long.class, Long.class, Long.class, Long.class, Long.class, Long.class, Long.class);
    set1.coGroup(set2).where(3, 0).equalTo(6, 0).sortFirstGroup(5, Order.DESCENDING).sortSecondGroup(1, Order.DESCENDING).sortSecondGroup(4, Order.ASCENDING).with(new IdentityCoGrouper<Tuple7<Long, Long, Long, Long, Long, Long, Long>>()).name("CoGroup").output(new DiscardingOutputFormat<Tuple7<Long, Long, Long, Long, Long, Long, Long>>()).name("Sink");
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan;
    try {
        oPlan = compileNoStats(plan);
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
        // silence the compiler
        return;
    }
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    SinkPlanNode sinkNode = resolver.getNode("Sink");
    DualInputPlanNode coGroupNode = resolver.getNode("CoGroup");
    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy());
    Channel c1 = coGroupNode.getInput1();
    Channel c2 = coGroupNode.getInput2();
    Assert.assertEquals(LocalStrategy.SORT, c1.getLocalStrategy());
    Assert.assertEquals(LocalStrategy.SORT, c2.getLocalStrategy());
    FieldList ship1 = new FieldList(3, 0);
    FieldList ship2 = new FieldList(6, 0);
    FieldList local1 = new FieldList(3, 0, 5);
    FieldList local2 = new FieldList(6, 0, 1, 4);
    Assert.assertEquals(ship1, c1.getShipStrategyKeys());
    Assert.assertEquals(ship2, c2.getShipStrategyKeys());
    Assert.assertEquals(local1, c1.getLocalStrategyKeys());
    Assert.assertEquals(local2, c2.getLocalStrategyKeys());
    Assert.assertTrue(c1.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
    Assert.assertTrue(c1.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
    // check that the local group orderings are correct
    Assert.assertEquals(false, c1.getLocalStrategySortOrder()[2]);
    Assert.assertEquals(false, c2.getLocalStrategySortOrder()[2]);
    Assert.assertEquals(true, c2.getLocalStrategySortOrder()[3]);
}

Example 50 with Channel

use of org.apache.flink.optimizer.plan.Channel in project flink by apache.

the class IterationsCompilerTest method testIterationNotPushingWorkOut.

@Test
public void testIterationNotPushingWorkOut() throws Exception {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(8);
        DataSet<Tuple2<Long, Long>> input1 = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
        DataSet<Tuple2<Long, Long>> input2 = env.readCsvFile("/some/file/path").types(Long.class, Long.class);
        // Use input1 as partial solution. Partial solution is used in a single join operation --> it is cheaper
        // to do the hash partitioning between the partial solution node and the join node
        // instead of pushing the partitioning out
        doSimpleBulkIteration(input1, input2).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        assertEquals(1, op.getDataSinks().size());
        assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof BulkIterationPlanNode);
        BulkIterationPlanNode bipn = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
        // check that work has not been pushed out
        for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) {
            assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy());
        }
        assertEquals(ShipStrategyType.FORWARD, bipn.getInput().getShipStrategy());
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}