Examples with Partitioner org.apache.flink.api.common.functions.Partitioner used on opensource projects

Example 6 with Partitioner

use of org.apache.flink.api.common.functions.Partitioner in project flink by apache.

the class BinaryCustomPartitioningCompatibilityTest method testCompatiblePartitioningCoGroup.

@Test
public void testCompatiblePartitioningCoGroup() {
    try {
        final Partitioner<Long> partitioner = new Partitioner<Long>() {

            @Override
            public int partition(Long key, int numPartitions) {
                return 0;
            }
        };
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
        input1.partitionCustom(partitioner, 1).coGroup(input2.partitionCustom(partitioner, 0)).where(1).equalTo(0).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode partitioner1 = (SingleInputPlanNode) coGroup.getInput1().getSource();
        SingleInputPlanNode partitioner2 = (SingleInputPlanNode) coGroup.getInput2().getSource();
        assertEquals(ShipStrategyType.FORWARD, coGroup.getInput1().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, coGroup.getInput2().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner1.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner2.getInput().getShipStrategy());
        assertEquals(partitioner, partitioner1.getInput().getPartitioner());
        assertEquals(partitioner, partitioner2.getInput().getPartitioner());
        new JobGraphGenerator().compileJobGraph(op);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with Partitioner

use of org.apache.flink.api.common.functions.Partitioner in project flink by apache.

the class CoGroupGlobalPropertiesCompatibilityTest method checkInompatiblePartitionings.

@Test
public void checkInompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1);
        final FieldList keysRight = new FieldList(3);
        final Partitioner<Object> part = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        final Partitioner<Object> part2 = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
        // test incompatible hash with custom partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible custom partitionings
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test incompatible range partitioning with different key size
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible range partitioning with different ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.DESCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist3 = new TestDistribution(1);
        TestDistribution dist4 = new TestDistribution(2);
        // test incompatible range partitioning with different distribution
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist3);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist4);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist3);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist4);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 8 with Partitioner

use of org.apache.flink.api.common.functions.Partitioner in project flink by apache.

the class PartitionOperatorTest method testPartitionCustomOperatorPreservesFields.

@Test
public void testPartitionCustomOperatorPreservesFields() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L)));
        data.partitionCustom(new Partitioner<Long>() {

            public int partition(Long key, int numPartitions) {
                return key.intValue();
            }
        }, 1).groupBy(1).reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode partitioner = (SingleInputPlanNode) reducer.getInput().getSource();
        assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with Partitioner

use of org.apache.flink.api.common.functions.Partitioner in project flink by apache.

the class JoinGlobalPropertiesCompatibilityTest method checkInompatiblePartitionings.

@Test
public void checkInompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1);
        final FieldList keysRight = new FieldList(3);
        final Partitioner<Object> part = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        final Partitioner<Object> part2 = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        SortMergeInnerJoinDescriptor descr = new SortMergeInnerJoinDescriptor(keysLeft, keysRight);
        // test incompatible hash with custom partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible custom partitionings
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test incompatible range partitioning with different key size
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible range partitioning with different ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.DESCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist3 = new TestDistribution(1);
        TestDistribution dist4 = new TestDistribution(2);
        // test incompatible range partitioning with different distribution
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist3);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist4);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist3);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist4);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with Partitioner

use of org.apache.flink.api.common.functions.Partitioner in project flink by apache.

the class PartitionerITCase method partitionerTest.

@Test
public void partitionerTest() {
    TestListResultSink<Tuple2<Integer, String>> hashPartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    TestListResultSink<Tuple2<Integer, String>> customPartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    TestListResultSink<Tuple2<Integer, String>> broadcastPartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    TestListResultSink<Tuple2<Integer, String>> forwardPartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    TestListResultSink<Tuple2<Integer, String>> rebalancePartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    TestListResultSink<Tuple2<Integer, String>> globalPartitionResultSink = new TestListResultSink<Tuple2<Integer, String>>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Tuple1<String>> src = env.fromElements(new Tuple1<String>("a"), new Tuple1<String>("b"), new Tuple1<String>("b"), new Tuple1<String>("a"), new Tuple1<String>("a"), new Tuple1<String>("c"), new Tuple1<String>("a"));
    // partition by hash
    src.keyBy(0).map(new SubtaskIndexAssigner()).addSink(hashPartitionResultSink);
    // partition custom
    DataStream<Tuple2<Integer, String>> partitionCustom = src.partitionCustom(new Partitioner<String>() {

        @Override
        public int partition(String key, int numPartitions) {
            if (key.equals("c")) {
                return 2;
            } else {
                return 0;
            }
        }
    }, 0).map(new SubtaskIndexAssigner());
    partitionCustom.addSink(customPartitionResultSink);
    // partition broadcast
    src.broadcast().map(new SubtaskIndexAssigner()).addSink(broadcastPartitionResultSink);
    // partition rebalance
    src.rebalance().map(new SubtaskIndexAssigner()).addSink(rebalancePartitionResultSink);
    // partition forward
    src.map(new MapFunction<Tuple1<String>, Tuple1<String>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple1<String> map(Tuple1<String> value) throws Exception {
            return value;
        }
    }).forward().map(new SubtaskIndexAssigner()).addSink(forwardPartitionResultSink);
    // partition global
    src.global().map(new SubtaskIndexAssigner()).addSink(globalPartitionResultSink);
    try {
        env.execute();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
    List<Tuple2<Integer, String>> hashPartitionResult = hashPartitionResultSink.getResult();
    List<Tuple2<Integer, String>> customPartitionResult = customPartitionResultSink.getResult();
    List<Tuple2<Integer, String>> broadcastPartitionResult = broadcastPartitionResultSink.getResult();
    List<Tuple2<Integer, String>> forwardPartitionResult = forwardPartitionResultSink.getResult();
    List<Tuple2<Integer, String>> rebalancePartitionResult = rebalancePartitionResultSink.getResult();
    List<Tuple2<Integer, String>> globalPartitionResult = globalPartitionResultSink.getResult();
    verifyHashPartitioning(hashPartitionResult);
    verifyCustomPartitioning(customPartitionResult);
    verifyBroadcastPartitioning(broadcastPartitionResult);
    verifyRebalancePartitioning(forwardPartitionResult);
    verifyRebalancePartitioning(rebalancePartitionResult);
    verifyGlobalPartitioning(globalPartitionResult);
}