use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.
the class LocalExecutor method executePlan.
/**
* Executes the given program on a local runtime and waits for the job to finish.
*
* <p>If the executor has not been started before, this starts the executor and shuts it down
* after the job finished. If the job runs in session mode, the executor is kept alive until
* no more references to the executor exist.</p>
*
* @param plan The plan of the program to execute.
* @return The net runtime of the program, in milliseconds.
*
* @throws Exception Thrown, if either the startup of the local execution context, or the execution
* caused an exception.
*/
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
if (plan == null) {
throw new IllegalArgumentException("The plan may not be null.");
}
synchronized (this.lock) {
// check if we start a session dedicated for this execution
final boolean shutDownAtEnd;
if (flink == null) {
shutDownAtEnd = true;
// configure the number of local slots equal to the parallelism of the local plan
if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
int maxParallelism = plan.getMaximumParallelism();
if (maxParallelism > 0) {
this.taskManagerNumSlots = maxParallelism;
}
}
// start the cluster for us
start();
} else {
// we use the existing session
shutDownAtEnd = false;
}
try {
Configuration configuration = this.flink.configuration();
Optimizer pc = new Optimizer(new DataStatistics(), configuration);
OptimizedPlan op = pc.compile(plan);
JobGraphGenerator jgg = new JobGraphGenerator(configuration);
JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());
boolean sysoutPrint = isPrintingStatusDuringExecution();
return flink.submitJobAndWait(jobGraph, sysoutPrint);
} finally {
if (shutDownAtEnd) {
stop();
}
}
}
}
use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.
the class ClusterClient method getJobGraph.
private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) {
JobGraph job;
if (optPlan instanceof StreamingPlan) {
job = ((StreamingPlan) optPlan).getJobGraph();
job.setSavepointRestoreSettings(savepointSettings);
} else {
JobGraphGenerator gen = new JobGraphGenerator(this.flinkConfig);
job = gen.compileJobGraph((OptimizedPlan) optPlan);
}
for (URL jar : jarFiles) {
try {
job.addJar(new Path(jar.toURI()));
} catch (URISyntaxException e) {
throw new RuntimeException("URL is invalid. This should not happen.", e);
}
}
job.setClasspaths(classpaths);
return job;
}
use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.
the class BranchingPlansCompilerTest method testBranchingWithMultipleDataSinksSmall.
/**
*
* <pre>
* (SRC A)
* / \
* (SINK A) (SINK B)
* </pre>
*/
@Test
public void testBranchingWithMultipleDataSinksSmall() {
try {
String outPath1 = "/tmp/out1";
String outPath2 = "/tmp/out2";
// construct the plan
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Long> source1 = env.generateSequence(0, 1);
source1.writeAsText(outPath1);
source1.writeAsText(outPath2);
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
// ---------- check the optimizer plan ----------
// number of sinks
Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
// sinks contain all sink paths
Set<String> allSinks = new HashSet<String>();
allSinks.add(outPath1);
allSinks.add(outPath2);
for (SinkPlanNode n : oPlan.getDataSinks()) {
String path = ((TextOutputFormat<String>) n.getSinkNode().getOperator().getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString();
Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
}
// ---------- compile plan to job graph to verify that no error is thrown ----------
JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.
the class BranchingPlansCompilerTest method testBranchEachContractType.
@SuppressWarnings("unchecked")
@Test
public void testBranchEachContractType() {
try {
// construct the plan
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Long> sourceA = env.generateSequence(0, 1);
DataSet<Long> sourceB = env.generateSequence(0, 1);
DataSet<Long> sourceC = env.generateSequence(0, 1);
DataSet<Long> map1 = sourceA.map(new IdentityMapper<Long>()).name("Map 1");
DataSet<Long> reduce1 = map1.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>()).name("Reduce 1");
DataSet<Long> join1 = sourceB.union(sourceB).union(sourceC).join(sourceC).where("*").equalTo("*").with(new IdentityJoiner<Long>()).name("Join 1");
DataSet<Long> coGroup1 = sourceA.coGroup(sourceB).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 1");
DataSet<Long> cross1 = reduce1.cross(coGroup1).with(new IdentityCrosser<Long>()).name("Cross 1");
DataSet<Long> coGroup2 = cross1.coGroup(cross1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 2");
DataSet<Long> coGroup3 = map1.coGroup(join1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 3");
DataSet<Long> map2 = coGroup3.map(new IdentityMapper<Long>()).name("Map 2");
DataSet<Long> coGroup4 = map2.coGroup(join1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 4");
DataSet<Long> coGroup5 = coGroup2.coGroup(coGroup1).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 5");
DataSet<Long> coGroup6 = reduce1.coGroup(coGroup4).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 6");
DataSet<Long> coGroup7 = coGroup5.coGroup(coGroup6).where("*").equalTo("*").with(new IdentityCoGrouper<Long>()).name("CoGroup 7");
coGroup7.union(sourceA).union(coGroup3).union(coGroup4).union(coGroup1).output(new DiscardingOutputFormat<Long>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
JobGraphGenerator jobGen = new JobGraphGenerator();
//Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
use of org.apache.flink.optimizer.plantranslate.JobGraphGenerator in project flink by apache.
the class BranchingPlansCompilerTest method testBranchingSourceMultipleTimes.
/**
* <pre>
* SINK
* |
* COGROUP
* +---/ \----+
* / \
* / MATCH10
* / | \
* / | MATCH9
* MATCH5 | | \
* | \ | | MATCH8
* | MATCH4 | | | \
* | | \ | | | MATCH7
* | | MATCH3 | | | | \
* | | | \ | | | | MATCH6
* | | | MATCH2 | | | | | |
* | | | | \ +--+--+--+--+--+
* | | | | MATCH1 MAP
* \ | | | | | /-----------/
* (DATA SOURCE ONE)
* </pre>
*/
@Test
public void testBranchingSourceMultipleTimes() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> mapped = source.map(new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
@Override
public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
return null;
}
});
DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
joined5.coGroup(joined10).where(1).equalTo(1).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Aggregations