use of org.apache.flink.api.java.ExecutionEnvironment in project flink by apache.
the class UnionITCase method testUnionWithEmptyDataSet.
@Test
public void testUnionWithEmptyDataSet() throws Exception {
/*
* Test on union with empty dataset
*/
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// Don't know how to make an empty result in an other way than filtering it
DataSet<Tuple3<Integer, Long, String>> empty = CollectionDataSets.get3TupleDataSet(env).filter(new RichFilter1());
DataSet<Tuple3<Integer, Long, String>> unionDs = CollectionDataSets.get3TupleDataSet(env).union(empty);
List<Tuple3<Integer, Long, String>> result = unionDs.collect();
String expected = FULL_TUPLE_3_STRING;
compareResultAsTuples(result, expected);
}
use of org.apache.flink.api.java.ExecutionEnvironment in project flink by apache.
the class UnionITCase method testUnion2IdenticalDataSets.
@Test
public void testUnion2IdenticalDataSets() throws Exception {
/*
* Union of 2 Same Data Sets
*/
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env));
List<Tuple3<Integer, Long, String>> result = unionDs.collect();
String expected = FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING;
compareResultAsTuples(result, expected);
}
use of org.apache.flink.api.java.ExecutionEnvironment in project flink by apache.
the class ReducePerformance method testReducePerformance.
private static <T, B extends CopyableIterator<T>> void testReducePerformance(B iterator, TypeInformation<T> typeInfo, CombineHint hint, int numRecords, boolean print) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.getConfig().enableObjectReuse();
@SuppressWarnings("unchecked") DataSet<T> output = env.fromParallelCollection(new SplittableRandomIterator<T, B>(numRecords, iterator), typeInfo).groupBy("0").reduce(new SumReducer()).setCombineHint(hint);
long start = System.currentTimeMillis();
System.out.println(output.count());
long end = System.currentTimeMillis();
if (print) {
System.out.println("=== Time for " + iterator.getClass().getSimpleName() + " with hint " + hint.toString() + ": " + (end - start) + "ms ===");
}
}
use of org.apache.flink.api.java.ExecutionEnvironment in project flink by apache.
the class AutoParallelismITCase method testProgramWithAutoParallelism.
@Test
public void testProgramWithAutoParallelism() {
try {
ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
env.setParallelism(ExecutionConfig.PARALLELISM_AUTO_MAX);
env.getConfig().disableSysoutLogging();
DataSet<Integer> result = env.createInput(new ParallelismDependentInputFormat()).rebalance().mapPartition(new ParallelismDependentMapPartition());
List<Integer> resultCollection = new ArrayList<Integer>();
result.output(new LocalCollectionOutputFormat<Integer>(resultCollection));
env.execute();
assertEquals(PARALLELISM, resultCollection.size());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
try {
cluster.stop();
} catch (Throwable t) {
// ignore exceptions on shutdown
}
}
}
use of org.apache.flink.api.java.ExecutionEnvironment in project flink by apache.
the class CustomPartitioningITCase method testProgram.
@Override
protected void testProgram() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
if (!isCollectionExecution()) {
Assert.assertTrue(env.getParallelism() > 1);
}
env.generateSequence(1, 1000).partitionCustom(new AllZeroPartitioner(), new IdKeySelector<Long>()).map(new FailExceptInPartitionZeroMapper()).output(new DiscardingOutputFormat<Long>());
env.execute();
}
Aggregations