Examples with GlobalProperties org.apache.flink.optimizer.dataproperties.GlobalProperties used on opensource projects

Example 21 with GlobalProperties

use of org.apache.flink.optimizer.dataproperties.GlobalProperties in project flink by apache.

the class PropertyDataSourceTest method checkSinglePartitionedSource7.

@Test
public void checkSinglePartitionedSource7() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSource<Tuple2<Long, String>> data = env.readCsvFile("/some/path").types(Long.class, String.class);
    data.getSplitDataProperties().splitsPartitionedBy("byDate", 1, 0);
    data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
    GlobalProperties gprops = sourceNode.getGlobalProperties();
    LocalProperties lprops = sourceNode.getLocalProperties();
    Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
    Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
    Assert.assertTrue(gprops.getCustomPartitioner() != null);
    Assert.assertTrue(lprops.getGroupedFields() == null);
    Assert.assertTrue(lprops.getOrdering() == null);
}

Example 22 with GlobalProperties

use of org.apache.flink.optimizer.dataproperties.GlobalProperties in project flink by apache.

the class PropertyDataSourceTest method checkSinglePartitionedOrderedSource4.

@Test
public void checkSinglePartitionedOrderedSource4() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSource<Tuple2<Long, String>> data = env.readCsvFile("/some/path").types(Long.class, String.class);
    data.getSplitDataProperties().splitsPartitionedBy(0, 1).splitsOrderedBy(new int[] { 1 }, new Order[] { Order.DESCENDING });
    data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
    GlobalProperties gprops = sourceNode.getGlobalProperties();
    LocalProperties lprops = sourceNode.getLocalProperties();
    Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
    Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
    Assert.assertTrue(lprops.getGroupedFields() == null);
    Assert.assertTrue(lprops.getOrdering() == null);
}

Example 23 with GlobalProperties

use of org.apache.flink.optimizer.dataproperties.GlobalProperties in project flink by apache.

the class PropertyDataSourceTest method checkSinglePartitionedGroupedSource2.

@Test
public void checkSinglePartitionedGroupedSource2() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSource<Tuple2<Long, String>> data = env.readCsvFile("/some/path").types(Long.class, String.class);
    data.getSplitDataProperties().splitsPartitionedBy(0).splitsGroupedBy(1, 0);
    data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
    GlobalProperties gprops = sourceNode.getGlobalProperties();
    LocalProperties lprops = sourceNode.getLocalProperties();
    Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
    Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
    Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0, 1)));
    Assert.assertTrue(lprops.getOrdering() == null);
}

Example 24 with GlobalProperties

use of org.apache.flink.optimizer.dataproperties.GlobalProperties in project flink by apache.

the class PropertyDataSourceTest method checkSinglePartitionedGroupedSource6.

@Test
public void checkSinglePartitionedGroupedSource6() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
    data.getSplitDataProperties().splitsPartitionedBy("f1.intField").splitsGroupedBy("f0; f1.intField");
    data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
    GlobalProperties gprops = sourceNode.getGlobalProperties();
    LocalProperties lprops = sourceNode.getLocalProperties();
    Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
    Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
    Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0, 2)));
    Assert.assertTrue(lprops.getOrdering() == null);
}

Example 25 with GlobalProperties

use of org.apache.flink.optimizer.dataproperties.GlobalProperties in project flink by apache.

the class PropertyDataSourceTest method checkCoPartitionedSources2.

@Test
public void checkCoPartitionedSources2() {
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSource<Tuple2<Long, String>> data1 = env.readCsvFile("/some/path").types(Long.class, String.class);
    data1.getSplitDataProperties().splitsPartitionedBy("byCountry", 0);
    DataSource<Tuple2<Long, String>> data2 = env.readCsvFile("/some/path").types(Long.class, String.class);
    data2.getSplitDataProperties().splitsPartitionedBy("byDate", 0);
    data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());
    Plan plan = env.createProgramPlan();
    // submit the plan to the compiler
    OptimizedPlan oPlan = compileNoStats(plan);
    // check the optimized Plan
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode) sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
    SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode) sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();
    GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
    LocalProperties lprops1 = sourceNode1.getLocalProperties();
    GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
    LocalProperties lprops2 = sourceNode2.getLocalProperties();
    Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
    Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
    Assert.assertTrue(lprops1.getGroupedFields() == null);
    Assert.assertTrue(lprops1.getOrdering() == null);
    Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
    Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
    Assert.assertTrue(lprops2.getGroupedFields() == null);
    Assert.assertTrue(lprops2.getOrdering() == null);
    Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}