use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.
the class JobSubmissionFailsITCase method testSubmitEmptyJobGraph.
@Test
public void testSubmitEmptyJobGraph() {
try {
final JobGraph jobGraph = new JobGraph("Testing job");
try {
submitJob(jobGraph);
fail("Expected JobSubmissionException.");
} catch (JobSubmissionException e) {
assertTrue(e.getMessage() != null && e.getMessage().contains("empty"));
} catch (Throwable t) {
t.printStackTrace();
fail("Caught wrong exception of type " + t.getClass() + ".");
}
cluster.submitJobAndWait(workingJobGraph, false);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.
the class JobSubmissionFailsITCase method setup.
@BeforeClass
public static void setup() {
try {
Configuration config = new Configuration();
config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS / 2);
cluster = new LocalFlinkMiniCluster(config);
cluster.start();
final JobVertex jobVertex = new JobVertex("Working job vertex.");
jobVertex.setInvokableClass(NoOpInvokable.class);
workingJobGraph = new JobGraph("Working testing job", jobVertex);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.
the class JobSubmissionFailsITCase method testExceptionInInitializeOnMaster.
@Test
public void testExceptionInInitializeOnMaster() {
try {
final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex");
failingJobVertex.setInvokableClass(NoOpInvokable.class);
final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex);
try {
submitJob(failingJobGraph);
fail("Expected JobExecutionException.");
} catch (JobExecutionException e) {
assertEquals("Test exception.", e.getCause().getMessage());
} catch (Throwable t) {
t.printStackTrace();
fail("Caught wrong exception of type " + t.getClass() + ".");
}
cluster.submitJobAndWait(workingJobGraph, false);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.
the class ExecutionGraphConstructionTest method testCoLocationConstraintCreation.
@Test
public void testCoLocationConstraintCreation() {
try {
final JobID jobId = new JobID();
final String jobName = "Co-Location Constraint Sample Job";
final Configuration cfg = new Configuration();
// simple group of two, cyclic
JobVertex v1 = new JobVertex("vertex1");
JobVertex v2 = new JobVertex("vertex2");
v1.setParallelism(6);
v2.setParallelism(4);
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
SlotSharingGroup sl1 = new SlotSharingGroup();
v1.setSlotSharingGroup(sl1);
v2.setSlotSharingGroup(sl1);
v2.setStrictlyCoLocatedWith(v1);
v1.setStrictlyCoLocatedWith(v2);
// complex forked dependency pattern
JobVertex v3 = new JobVertex("vertex3");
JobVertex v4 = new JobVertex("vertex4");
JobVertex v5 = new JobVertex("vertex5");
JobVertex v6 = new JobVertex("vertex6");
JobVertex v7 = new JobVertex("vertex7");
v3.setParallelism(3);
v4.setParallelism(3);
v5.setParallelism(3);
v6.setParallelism(3);
v7.setParallelism(3);
v3.setInvokableClass(AbstractInvokable.class);
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
v6.setInvokableClass(AbstractInvokable.class);
v7.setInvokableClass(AbstractInvokable.class);
SlotSharingGroup sl2 = new SlotSharingGroup();
v3.setSlotSharingGroup(sl2);
v4.setSlotSharingGroup(sl2);
v5.setSlotSharingGroup(sl2);
v6.setSlotSharingGroup(sl2);
v7.setSlotSharingGroup(sl2);
v4.setStrictlyCoLocatedWith(v3);
v5.setStrictlyCoLocatedWith(v4);
v6.setStrictlyCoLocatedWith(v3);
v3.setStrictlyCoLocatedWith(v7);
// isolated vertex
JobVertex v8 = new JobVertex("vertex8");
v8.setParallelism(2);
v8.setInvokableClass(AbstractInvokable.class);
JobGraph jg = new JobGraph(jobId, jobName, v1, v2, v3, v4, v5, v6, v7, v8);
ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
eg.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
// check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
{
ExecutionVertex[] v1s = eg.getJobVertex(v1.getID()).getTaskVertices();
ExecutionVertex[] v2s = eg.getJobVertex(v2.getID()).getTaskVertices();
Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
for (int i = 0; i < v2.getParallelism(); i++) {
assertNotNull(v1s[i].getLocationConstraint());
assertNotNull(v2s[i].getLocationConstraint());
assertTrue(v1s[i].getLocationConstraint() == v2s[i].getLocationConstraint());
all.add(v1s[i].getLocationConstraint());
}
for (int i = v2.getParallelism(); i < v1.getParallelism(); i++) {
assertNotNull(v1s[i].getLocationConstraint());
all.add(v1s[i].getLocationConstraint());
}
assertEquals("not all co location constraints are distinct", v1.getParallelism(), all.size());
}
// check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
{
ExecutionVertex[] v3s = eg.getJobVertex(v3.getID()).getTaskVertices();
ExecutionVertex[] v4s = eg.getJobVertex(v4.getID()).getTaskVertices();
ExecutionVertex[] v5s = eg.getJobVertex(v5.getID()).getTaskVertices();
ExecutionVertex[] v6s = eg.getJobVertex(v6.getID()).getTaskVertices();
ExecutionVertex[] v7s = eg.getJobVertex(v7.getID()).getTaskVertices();
Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
for (int i = 0; i < v3.getParallelism(); i++) {
assertNotNull(v3s[i].getLocationConstraint());
assertTrue(v3s[i].getLocationConstraint() == v4s[i].getLocationConstraint());
assertTrue(v4s[i].getLocationConstraint() == v5s[i].getLocationConstraint());
assertTrue(v5s[i].getLocationConstraint() == v6s[i].getLocationConstraint());
assertTrue(v6s[i].getLocationConstraint() == v7s[i].getLocationConstraint());
all.add(v3s[i].getLocationConstraint());
}
assertEquals("not all co location constraints are distinct", v3.getParallelism(), all.size());
}
// check the v8 has no co location hints
{
ExecutionVertex[] v8s = eg.getJobVertex(v8.getID()).getTaskVertices();
for (int i = 0; i < v8.getParallelism(); i++) {
assertNull(v8s[i].getLocationConstraint());
}
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.
the class CoordinatorShutdownTest method testCoordinatorShutsDownOnSuccess.
@Test
public void testCoordinatorShutsDownOnSuccess() {
LocalFlinkMiniCluster cluster = null;
try {
Configuration config = new Configuration();
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
cluster = new LocalFlinkMiniCluster(config, true);
cluster.start();
// build a test graph with snapshotting enabled
JobVertex vertex = new JobVertex("Test Vertex");
vertex.setInvokableClass(BlockingInvokable.class);
List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
JobGraph testGraph = new JobGraph("test job", vertex);
testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), null, true));
ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
FiniteDuration timeout = new FiniteDuration(60, TimeUnit.SECONDS);
JobManagerMessages.SubmitJob submitMessage = new JobManagerMessages.SubmitJob(testGraph, ListeningBehaviour.EXECUTION_RESULT);
// submit is successful, but then the job blocks due to the invokable
Future<Object> submitFuture = jmGateway.ask(submitMessage, timeout);
Await.result(submitFuture, timeout);
// get the execution graph and store the ExecutionGraph reference
Future<Object> jobRequestFuture = jmGateway.ask(new JobManagerMessages.RequestJob(testGraph.getJobID()), timeout);
ExecutionGraph graph = (ExecutionGraph) ((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
assertNotNull(graph);
BlockingInvokable.unblock();
graph.waitUntilFinished();
// verify that the coordinator was shut down
CheckpointCoordinator coord = graph.getCheckpointCoordinator();
assertTrue(coord == null || coord.isShutdown());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
cluster.awaitTermination();
}
}
}
Aggregations