use of org.apache.flink.runtime.executiongraph.IntermediateResultPartition in project flink by apache.
the class JobManagerTest method testRequestPartitionStateUnregisteredExecution.
/**
* Tests the JobManager response when the execution is not registered with
* the ExecutionGraph.
*/
@Test
public void testRequestPartitionStateUnregisteredExecution() throws Exception {
new JavaTestKit(system) {
{
new Within(duration("15 seconds")) {
@Override
protected void run() {
// Setup
TestingCluster cluster = null;
try {
cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
final IntermediateDataSetID rid = new IntermediateDataSetID();
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
// just finish
sender.setInvokableClass(NoOpInvokable.class);
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobVertex sender2 = new JobVertex("Blocking Sender");
sender2.setParallelism(1);
// just block
sender2.setInvokableClass(BlockingNoOpInvokable.class);
sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
final JobID jid = jobGraph.getJobID();
final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
// we can set the leader session ID to None because we don't use this gateway to send messages
final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
// Submit the job and wait for all vertices to be running
jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
expectMsgClass(JobSubmitSuccess.class);
jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
expectMsgClass(AllVerticesRunning.class);
Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
while (vertex.getExecutionState() != ExecutionState.FINISHED) {
Thread.sleep(1);
}
IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
// Producer finished, request state
Object request = new RequestPartitionProducerState(jid, rid, partitionId);
Future<ExecutionState> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime()).mapTo(ClassTag$.MODULE$.<ExecutionState>apply(ExecutionState.class));
assertEquals(ExecutionState.FINISHED, Await.result(producerStateFuture, getRemainingTime()));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
};
}
};
}
use of org.apache.flink.runtime.executiongraph.IntermediateResultPartition in project flink by apache.
the class JobManagerTest method testRequestPartitionState.
/**
* Tests responses to partition state requests.
*/
@Test
public void testRequestPartitionState() throws Exception {
new JavaTestKit(system) {
{
new Within(duration("15 seconds")) {
@Override
protected void run() {
// Setup
TestingCluster cluster = null;
try {
cluster = startTestingCluster(2, 1, DEFAULT_AKKA_ASK_TIMEOUT());
final IntermediateDataSetID rid = new IntermediateDataSetID();
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
// just block
sender.setInvokableClass(BlockingNoOpInvokable.class);
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobGraph jobGraph = new JobGraph("Blocking test job", sender);
final JobID jid = jobGraph.getJobID();
final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
// we can set the leader session ID to None because we don't use this gateway to send messages
final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
// Submit the job and wait for all vertices to be running
jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
expectMsgClass(JobSubmitSuccess.class);
jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
expectMsgClass(AllVerticesRunning.class);
// This is the mock execution ID of the task requesting the state of the partition
final ExecutionAttemptID receiver = new ExecutionAttemptID();
// Request the execution graph to get the runtime info
jobManagerGateway.tell(new RequestExecutionGraph(jid), testActorGateway);
final ExecutionGraph eg = (ExecutionGraph) expectMsgClass(ExecutionGraphFound.class).executionGraph();
final ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
final IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
// - The test ----------------------------------------------------------------------
// 1. All execution states
RequestPartitionProducerState request = new RequestPartitionProducerState(jid, rid, partitionId);
for (ExecutionState state : ExecutionState.values()) {
ExecutionGraphTestUtils.setVertexState(vertex, state);
Future<ExecutionState> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime()).mapTo(ClassTag$.MODULE$.<ExecutionState>apply(ExecutionState.class));
ExecutionState resp = Await.result(futurePartitionState, getRemainingTime());
assertEquals(state, resp);
}
// 2. Non-existing execution
request = new RequestPartitionProducerState(jid, rid, new ResultPartitionID());
Future<?> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(futurePartitionState, getRemainingTime());
fail("Did not fail with expected RuntimeException");
} catch (RuntimeException e) {
assertEquals(IllegalArgumentException.class, e.getCause().getClass());
}
// 3. Non-existing job
request = new RequestPartitionProducerState(new JobID(), rid, new ResultPartitionID());
futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(futurePartitionState, getRemainingTime());
fail("Did not fail with expected IllegalArgumentException");
} catch (IllegalArgumentException ignored) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
};
}
};
}
use of org.apache.flink.runtime.executiongraph.IntermediateResultPartition in project flink by apache.
the class JobManagerTest method testRequestPartitionStateMoreRecentExecutionAttempt.
/**
* Tests the JobManager response when the execution is not registered with
* the ExecutionGraph anymore and a new execution attempt is available.
*/
@Test
public void testRequestPartitionStateMoreRecentExecutionAttempt() throws Exception {
new JavaTestKit(system) {
{
new Within(duration("15 seconds")) {
@Override
protected void run() {
// Setup
TestingCluster cluster = null;
try {
cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
final IntermediateDataSetID rid = new IntermediateDataSetID();
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
// just finish
sender.setInvokableClass(NoOpInvokable.class);
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobVertex sender2 = new JobVertex("Blocking Sender");
sender2.setParallelism(1);
// just block
sender2.setInvokableClass(BlockingNoOpInvokable.class);
sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
final JobID jid = jobGraph.getJobID();
final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
// we can set the leader session ID to None because we don't use this gateway to send messages
final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
// Submit the job and wait for all vertices to be running
jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
expectMsgClass(JobManagerMessages.JobSubmitSuccess.class);
jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
expectMsgClass(TestingJobManagerMessages.AllVerticesRunning.class);
Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
while (vertex.getExecutionState() != ExecutionState.FINISHED) {
Thread.sleep(1);
}
IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
// Reset execution => new execution attempt
vertex.resetForNewExecution();
// Producer finished, request state
Object request = new JobManagerMessages.RequestPartitionProducerState(jid, rid, partitionId);
Future<?> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime());
try {
Await.result(producerStateFuture, getRemainingTime());
fail("Did not fail with expected Exception");
} catch (PartitionProducerDisposedException ignored) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
};
}
};
}
use of org.apache.flink.runtime.executiongraph.IntermediateResultPartition in project flink by apache.
the class InputChannelDeploymentDescriptorTest method testMixedLocalRemoteUnknownDeployment.
/**
* Tests the deployment descriptors for local, remote, and unknown partition
* locations (with lazy deployment allowed and all execution states for the
* producers).
*/
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
boolean allowLazyDeployment = true;
ResourceID consumerResourceId = ResourceID.generate();
ExecutionVertex consumer = mock(ExecutionVertex.class);
SimpleSlot consumerSlot = mockSlot(consumerResourceId);
// states.
for (ExecutionState state : ExecutionState.values()) {
// Local partition
ExecutionVertex localProducer = mockExecutionVertex(state, consumerResourceId);
IntermediateResultPartition localPartition = mockPartition(localProducer);
ResultPartitionID localPartitionId = new ResultPartitionID(localPartition.getPartitionId(), localProducer.getCurrentExecutionAttempt().getAttemptId());
ExecutionEdge localEdge = new ExecutionEdge(localPartition, consumer, 0);
// Remote partition
// new resource ID
ExecutionVertex remoteProducer = mockExecutionVertex(state, ResourceID.generate());
IntermediateResultPartition remotePartition = mockPartition(remoteProducer);
ResultPartitionID remotePartitionId = new ResultPartitionID(remotePartition.getPartitionId(), remoteProducer.getCurrentExecutionAttempt().getAttemptId());
ConnectionID remoteConnectionId = new ConnectionID(remoteProducer.getCurrentAssignedResource().getTaskManagerLocation(), 0);
ExecutionEdge remoteEdge = new ExecutionEdge(remotePartition, consumer, 1);
// Unknown partition
// no assigned resource
ExecutionVertex unknownProducer = mockExecutionVertex(state, null);
IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { localEdge, remoteEdge, unknownEdge }, consumerSlot, allowLazyDeployment);
assertEquals(3, desc.length);
// These states are allowed
if (state == ExecutionState.RUNNING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
// Create local or remote channels
assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
assertTrue(desc[0].getConsumedPartitionLocation().isLocal());
assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
assertTrue(desc[1].getConsumedPartitionLocation().isRemote());
assertEquals(remoteConnectionId, desc[1].getConsumedPartitionLocation().getConnectionId());
} else {
// Unknown (lazy deployment allowed)
assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
assertTrue(desc[1].getConsumedPartitionLocation().isUnknown());
assertNull(desc[1].getConsumedPartitionLocation().getConnectionId());
}
assertEquals(unknownPartitionId, desc[2].getConsumedPartitionId());
assertTrue(desc[2].getConsumedPartitionLocation().isUnknown());
assertNull(desc[2].getConsumedPartitionLocation().getConnectionId());
}
}
use of org.apache.flink.runtime.executiongraph.IntermediateResultPartition in project flink by apache.
the class InputChannelDeploymentDescriptorTest method testUnknownChannelWithoutLazyDeploymentThrows.
@Test
public void testUnknownChannelWithoutLazyDeploymentThrows() throws Exception {
ResourceID consumerResourceId = ResourceID.generate();
ExecutionVertex consumer = mock(ExecutionVertex.class);
SimpleSlot consumerSlot = mockSlot(consumerResourceId);
// Unknown partition
// no assigned resource
ExecutionVertex unknownProducer = mockExecutionVertex(ExecutionState.CREATED, null);
IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
// This should work if lazy deployment is allowed
boolean allowLazyDeployment = true;
InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
assertEquals(1, desc.length);
assertEquals(unknownPartitionId, desc[0].getConsumedPartitionId());
assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
try {
// Fail if lazy deployment is *not* allowed
allowLazyDeployment = false;
InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
fail("Did not throw expected ExecutionGraphException");
} catch (ExecutionGraphException ignored) {
}
}
Aggregations