use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.
the class JobManagerTest method testRequestPartitionState.
/**
* Tests responses to partition state requests.
*/
@Test
public void testRequestPartitionState() throws Exception {
new JavaTestKit(system) {
{
new Within(duration("15 seconds")) {
@Override
protected void run() {
// Setup
TestingCluster cluster = null;
try {
cluster = startTestingCluster(2, 1, DEFAULT_AKKA_ASK_TIMEOUT());
final IntermediateDataSetID rid = new IntermediateDataSetID();
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
// just block
sender.setInvokableClass(BlockingNoOpInvokable.class);
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobGraph jobGraph = new JobGraph("Blocking test job", sender);
final JobID jid = jobGraph.getJobID();
final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
// we can set the leader session ID to None because we don't use this gateway to send messages
final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
// Submit the job and wait for all vertices to be running
jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
expectMsgClass(JobSubmitSuccess.class);
jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
expectMsgClass(AllVerticesRunning.class);
// This is the mock execution ID of the task requesting the state of the partition
final ExecutionAttemptID receiver = new ExecutionAttemptID();
// Request the execution graph to get the runtime info
jobManagerGateway.tell(new RequestExecutionGraph(jid), testActorGateway);
final ExecutionGraph eg = (ExecutionGraph) expectMsgClass(ExecutionGraphFound.class).executionGraph();
final ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
final IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
// - The test ----------------------------------------------------------------------
// 1. All execution states
RequestPartitionProducerState request = new RequestPartitionProducerState(jid, rid, partitionId);
for (ExecutionState state : ExecutionState.values()) {
ExecutionGraphTestUtils.setVertexState(vertex, state);
Future<ExecutionState> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime()).mapTo(ClassTag$.MODULE$.<ExecutionState>apply(ExecutionState.class));
ExecutionState resp = Await.result(futurePartitionState, getRemainingTime());
assertEquals(state, resp);
}
// 2. Non-existing execution
request = new RequestPartitionProducerState(jid, rid, new ResultPartitionID());
Future<?> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(futurePartitionState, getRemainingTime());
fail("Did not fail with expected RuntimeException");
} catch (RuntimeException e) {
assertEquals(IllegalArgumentException.class, e.getCause().getClass());
}
// 3. Non-existing job
request = new RequestPartitionProducerState(new JobID(), rid, new ResultPartitionID());
futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(futurePartitionState, getRemainingTime());
fail("Did not fail with expected IllegalArgumentException");
} catch (IllegalArgumentException ignored) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
};
}
};
}
use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.
the class JobManagerTest method testRequestPartitionStateMoreRecentExecutionAttempt.
/**
* Tests the JobManager response when the execution is not registered with
* the ExecutionGraph anymore and a new execution attempt is available.
*/
@Test
public void testRequestPartitionStateMoreRecentExecutionAttempt() throws Exception {
new JavaTestKit(system) {
{
new Within(duration("15 seconds")) {
@Override
protected void run() {
// Setup
TestingCluster cluster = null;
try {
cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
final IntermediateDataSetID rid = new IntermediateDataSetID();
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
// just finish
sender.setInvokableClass(NoOpInvokable.class);
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobVertex sender2 = new JobVertex("Blocking Sender");
sender2.setParallelism(1);
// just block
sender2.setInvokableClass(BlockingNoOpInvokable.class);
sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
final JobID jid = jobGraph.getJobID();
final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
// we can set the leader session ID to None because we don't use this gateway to send messages
final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
// Submit the job and wait for all vertices to be running
jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
expectMsgClass(JobManagerMessages.JobSubmitSuccess.class);
jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
expectMsgClass(TestingJobManagerMessages.AllVerticesRunning.class);
Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
while (vertex.getExecutionState() != ExecutionState.FINISHED) {
Thread.sleep(1);
}
IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
// Reset execution => new execution attempt
vertex.resetForNewExecution();
// Producer finished, request state
Object request = new JobManagerMessages.RequestPartitionProducerState(jid, rid, partitionId);
Future<?> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(producerStateFuture, getRemainingTime());
fail("Did not fail with expected Exception");
} catch (PartitionProducerDisposedException ignored) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
};
}
};
}
use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.
the class JobCancellationWithSavepointHandlersTest method testSavepointDirectoryConfiguration.
/**
* Tests that the savepoint directory configuration is respected.
*/
@Test
public void testSavepointDirectoryConfiguration() throws Exception {
long timeout = 128288238L;
JobID jobId = new JobID();
ExecutionGraphHolder holder = mock(ExecutionGraphHolder.class);
ExecutionGraph graph = mock(ExecutionGraph.class);
CheckpointCoordinator coord = mock(CheckpointCoordinator.class);
when(holder.getExecutionGraph(eq(jobId), any(ActorGateway.class))).thenReturn(graph);
when(graph.getCheckpointCoordinator()).thenReturn(coord);
when(coord.getCheckpointTimeout()).thenReturn(timeout);
JobCancellationWithSavepointHandlers handlers = new JobCancellationWithSavepointHandlers(holder, EC, "the-default-directory");
JobCancellationWithSavepointHandlers.TriggerHandler handler = handlers.getTriggerHandler();
Map<String, String> params = new HashMap<>();
params.put("jobid", jobId.toString());
ActorGateway jobManager = mock(ActorGateway.class);
Future<Object> future = Futures.successful((Object) new CancellationSuccess(jobId, null));
when(jobManager.ask(any(Object.class), any(FiniteDuration.class))).thenReturn(future);
// 1. Use targetDirectory path param
params.put("targetDirectory", "custom-directory");
handler.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
verify(jobManager).ask(eq(new CancelJobWithSavepoint(jobId, "custom-directory")), eq(FiniteDuration.apply(timeout, "ms")));
// 2. Use default
params.remove("targetDirectory");
handler.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
verify(jobManager).ask(eq(new CancelJobWithSavepoint(jobId, "the-default-directory")), eq(FiniteDuration.apply(timeout, "ms")));
// 3. Throw Exception
handlers = new JobCancellationWithSavepointHandlers(holder, EC, null);
handler = handlers.getTriggerHandler();
try {
handler.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
fail("Did not throw expected test Exception");
} catch (Exception e) {
IllegalStateException cause = (IllegalStateException) e.getCause();
assertEquals(true, cause.getMessage().contains(ConfigConstants.SAVEPOINT_DIRECTORY_KEY));
}
}
use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.
the class JobCancellationWithSavepointHandlersTest method testAskTimeoutEqualsCheckpointTimeout.
/**
* Tests that the cancellation ask timeout respects the checkpoint timeout.
* Otherwise, AskTimeoutExceptions are bound to happen for large state.
*/
@Test
public void testAskTimeoutEqualsCheckpointTimeout() throws Exception {
long timeout = 128288238L;
JobID jobId = new JobID();
ExecutionGraphHolder holder = mock(ExecutionGraphHolder.class);
ExecutionGraph graph = mock(ExecutionGraph.class);
CheckpointCoordinator coord = mock(CheckpointCoordinator.class);
when(holder.getExecutionGraph(eq(jobId), any(ActorGateway.class))).thenReturn(graph);
when(graph.getCheckpointCoordinator()).thenReturn(coord);
when(coord.getCheckpointTimeout()).thenReturn(timeout);
JobCancellationWithSavepointHandlers handlers = new JobCancellationWithSavepointHandlers(holder, EC);
JobCancellationWithSavepointHandlers.TriggerHandler handler = handlers.getTriggerHandler();
Map<String, String> params = new HashMap<>();
params.put("jobid", jobId.toString());
params.put("targetDirectory", "placeholder");
ActorGateway jobManager = mock(ActorGateway.class);
Future<Object> future = Futures.successful((Object) new CancellationSuccess(jobId, null));
when(jobManager.ask(any(Object.class), any(FiniteDuration.class))).thenReturn(future);
handler.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
verify(jobManager).ask(any(CancelJobWithSavepoint.class), eq(FiniteDuration.apply(timeout, "ms")));
}
use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.
the class JobCancellationWithSavepointHandlersTest method testFailedCancellation.
/**
* Tests response when a request fails.
*/
@Test
public void testFailedCancellation() throws Exception {
JobID jobId = new JobID();
ExecutionGraphHolder holder = mock(ExecutionGraphHolder.class);
ExecutionGraph graph = mock(ExecutionGraph.class);
CheckpointCoordinator coord = mock(CheckpointCoordinator.class);
when(holder.getExecutionGraph(eq(jobId), any(ActorGateway.class))).thenReturn(graph);
when(graph.getCheckpointCoordinator()).thenReturn(coord);
JobCancellationWithSavepointHandlers handlers = new JobCancellationWithSavepointHandlers(holder, EC);
JobCancellationWithSavepointHandlers.TriggerHandler trigger = handlers.getTriggerHandler();
JobCancellationWithSavepointHandlers.InProgressHandler progress = handlers.getInProgressHandler();
Map<String, String> params = new HashMap<>();
params.put("jobid", jobId.toString());
params.put("targetDirectory", "custom-directory");
ActorGateway jobManager = mock(ActorGateway.class);
// Successful
Future<Object> future = Futures.failed(new Exception("Test Exception"));
when(jobManager.ask(any(Object.class), any(FiniteDuration.class))).thenReturn(future);
// Trigger
trigger.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
verify(jobManager).ask(eq(new CancelJobWithSavepoint(jobId, "custom-directory")), any(FiniteDuration.class));
// Query progress
params.put("requestId", "1");
FullHttpResponse response = progress.handleRequest(params, Collections.<String, String>emptyMap(), jobManager);
assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, response.getStatus());
assertEquals("application/json", response.headers().get(HttpHeaders.Names.CONTENT_TYPE));
assertEquals(Integer.toString(response.content().readableBytes()), response.headers().get(HttpHeaders.Names.CONTENT_LENGTH));
String json = response.content().toString(Charset.forName("UTF-8"));
JsonNode root = new ObjectMapper().readTree(json);
assertEquals("failed", root.get("status").getValueAsText());
assertEquals("1", root.get("request-id").getValueAsText());
assertEquals("Test Exception", root.get("cause").getValueAsText());
}
Aggregations