use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class GroupReduceCompilationTest method testGroupedReduceWithSelectorFunctionKeyNoncombinable.
@Test
public void testGroupedReduceWithSelectorFunctionKeyNoncombinable() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(8);
DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class).name("source").setParallelism(6);
data.groupBy(new KeySelector<Tuple2<String, Double>, String>() {
public String getKey(Tuple2<String, Double> value) {
return value.f0;
}
}).reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {
}
}).name("reducer").output(new DiscardingOutputFormat<Tuple2<String, Double>>()).name("sink");
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
// get the original nodes
SourcePlanNode sourceNode = resolver.getNode("source");
SingleInputPlanNode reduceNode = resolver.getNode("reducer");
SinkPlanNode sinkNode = resolver.getNode("sink");
// get the key extractors and projectors
SingleInputPlanNode keyExtractor = (SingleInputPlanNode) reduceNode.getInput().getSource();
SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
// check wiring
assertEquals(sourceNode, keyExtractor.getInput().getSource());
assertEquals(keyProjector, sinkNode.getInput().getSource());
// check that both reduce and combiner have the same strategy
assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
// check the keys
assertEquals(new FieldList(0), reduceNode.getKeys(0));
assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
// check parallelism
assertEquals(6, sourceNode.getParallelism());
assertEquals(6, keyExtractor.getParallelism());
assertEquals(8, reduceNode.getParallelism());
assertEquals(8, keyProjector.getParallelism());
assertEquals(8, sinkNode.getParallelism());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
}
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class GlobalPropertiesFilteringTest method testAnyPartitioningErased.
@Test
public void testAnyPartitioningErased() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0;1" }, null, null, tupleInfo, tupleInfo);
GlobalProperties gprops = new GlobalProperties();
gprops.setAnyPartitioning(new FieldList(0, 1, 4));
GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
assertNull(result.getPartitioningFields());
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class GlobalPropertiesFilteringTest method testInvalidInputIndex.
@Test(expected = IndexOutOfBoundsException.class)
public void testInvalidInputIndex() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0;1" }, null, null, tupleInfo, tupleInfo);
GlobalProperties gprops = new GlobalProperties();
gprops.setHashPartitioned(new FieldList(0, 1));
gprops.filterBySemanticProperties(sprops, 1);
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class GlobalPropertiesFilteringTest method testAnyPartitioningPreserved1.
@Test
public void testAnyPartitioningPreserved1() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0;1;4" }, null, null, tupleInfo, tupleInfo);
GlobalProperties gprops = new GlobalProperties();
gprops.setAnyPartitioning(new FieldList(0, 1, 4));
GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
FieldList pFields = result.getPartitioningFields();
assertEquals(3, pFields.size());
assertTrue(pFields.contains(0));
assertTrue(pFields.contains(1));
assertTrue(pFields.contains(4));
}
use of org.apache.flink.api.common.operators.util.FieldList in project flink by apache.
the class GlobalPropertiesFilteringTest method testCustomPartitioningPreserved1.
@Test
public void testCustomPartitioningPreserved1() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "0;1;4" }, null, null, tupleInfo, tupleInfo);
GlobalProperties gprops = new GlobalProperties();
Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
gprops.setCustomPartitioned(new FieldList(0, 4), myP);
GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
FieldList pFields = result.getPartitioningFields();
assertEquals(2, pFields.size());
assertTrue(pFields.contains(0));
assertTrue(pFields.contains(4));
assertEquals(myP, result.getCustomPartitioner());
}
Aggregations