use of org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture in project flink by apache.
the class ExecutionGraphUtilsTest method testReleaseSlots.
@Test
public void testReleaseSlots() {
final JobID jid = new JobID();
final SlotOwner owner = mock(SlotOwner.class);
final SimpleSlot slot1 = new SimpleSlot(createAllocatedSlot(jid, 0), owner, 0);
final SimpleSlot slot2 = new SimpleSlot(createAllocatedSlot(jid, 1), owner, 1);
final SimpleSlot slot3 = new SimpleSlot(createAllocatedSlot(jid, 2), owner, 2);
final FlinkCompletableFuture<SimpleSlot> incompleteFuture = new FlinkCompletableFuture<>();
final FlinkCompletableFuture<SimpleSlot> completeFuture = new FlinkCompletableFuture<>();
completeFuture.complete(slot2);
final FlinkCompletableFuture<SimpleSlot> disposedSlotFuture = new FlinkCompletableFuture<>();
slot3.releaseSlot();
disposedSlotFuture.complete(slot3);
// release all futures
ExecutionGraphUtils.releaseSlotFuture(incompleteFuture);
ExecutionGraphUtils.releaseSlotFuture(completeFuture);
ExecutionGraphUtils.releaseSlotFuture(disposedSlotFuture);
// only now complete the incomplete future
incompleteFuture.complete(slot1);
// verify that each slot was returned once to the owner
verify(owner, times(1)).returnAllocatedSlot(eq(slot1));
verify(owner, times(1)).returnAllocatedSlot(eq(slot2));
verify(owner, times(1)).returnAllocatedSlot(eq(slot3));
}
use of org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture in project flink by apache.
the class ExecutionGraphSchedulingTest method testScheduleSourceBeforeTarget.
// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
/**
* Tests that with scheduling futures and pipelined deployment, the target vertex will
* not deploy its task before the source vertex does.
*/
@Test
public void testScheduleSourceBeforeTarget() throws Exception {
// [pipelined]
// we construct a simple graph (source) ----------------> (target)
final int parallelism = 1;
final JobVertex sourceVertex = new JobVertex("source");
sourceVertex.setParallelism(parallelism);
sourceVertex.setInvokableClass(NoOpInvokable.class);
final JobVertex targetVertex = new JobVertex("target");
targetVertex.setParallelism(parallelism);
targetVertex.setInvokableClass(NoOpInvokable.class);
targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
final JobID jobId = new JobID();
final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
final FlinkCompletableFuture<SimpleSlot> sourceFuture = new FlinkCompletableFuture<>();
final FlinkCompletableFuture<SimpleSlot> targetFuture = new FlinkCompletableFuture<>();
ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
slotProvider.addSlot(sourceVertex.getID(), 0, sourceFuture);
slotProvider.addSlot(targetVertex.getID(), 0, targetFuture);
final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
// set up two TaskManager gateways and slots
final TaskManagerGateway gatewaySource = createTaskManager();
final TaskManagerGateway gatewayTarget = createTaskManager();
final SimpleSlot sourceSlot = createSlot(gatewaySource, jobId);
final SimpleSlot targetSlot = createSlot(gatewayTarget, jobId);
eg.setScheduleMode(ScheduleMode.EAGER);
eg.setQueuedSchedulingAllowed(true);
eg.scheduleForExecution();
// job should be running
assertEquals(JobStatus.RUNNING, eg.getState());
// we fulfill the target slot before the source slot
// that should not cause a deployment or deployment related failure
targetFuture.complete(targetSlot);
verify(gatewayTarget, new Timeout(50, times(0))).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
assertEquals(JobStatus.RUNNING, eg.getState());
// now supply the source slot
sourceFuture.complete(sourceSlot);
// by now, all deployments should have happened
verify(gatewaySource, timeout(1000)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
verify(gatewayTarget, timeout(1000)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
assertEquals(JobStatus.RUNNING, eg.getState());
}
use of org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture in project flink by apache.
the class ExecutionGraphSchedulingTest method testDeployPipelinedConnectedComponentsTogether.
/**
* This test verifies that before deploying a pipelined connected component, the
* full set of slots is available, and that not some tasks are deployed, and later the
* system realizes that not enough resources are available.
*/
@Test
public void testDeployPipelinedConnectedComponentsTogether() throws Exception {
// [pipelined]
// we construct a simple graph (source) ----------------> (target)
final int parallelism = 8;
final JobVertex sourceVertex = new JobVertex("source");
sourceVertex.setParallelism(parallelism);
sourceVertex.setInvokableClass(NoOpInvokable.class);
final JobVertex targetVertex = new JobVertex("target");
targetVertex.setParallelism(parallelism);
targetVertex.setInvokableClass(NoOpInvokable.class);
targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
final JobID jobId = new JobID();
final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
@SuppressWarnings({ "unchecked", "rawtypes" }) final FlinkCompletableFuture<SimpleSlot>[] sourceFutures = new FlinkCompletableFuture[parallelism];
@SuppressWarnings({ "unchecked", "rawtypes" }) final FlinkCompletableFuture<SimpleSlot>[] targetFutures = new FlinkCompletableFuture[parallelism];
//
// Create the slots, futures, and the slot provider
final TaskManagerGateway[] sourceTaskManagers = new TaskManagerGateway[parallelism];
final TaskManagerGateway[] targetTaskManagers = new TaskManagerGateway[parallelism];
final SimpleSlot[] sourceSlots = new SimpleSlot[parallelism];
final SimpleSlot[] targetSlots = new SimpleSlot[parallelism];
for (int i = 0; i < parallelism; i++) {
sourceTaskManagers[i] = createTaskManager();
targetTaskManagers[i] = createTaskManager();
sourceSlots[i] = createSlot(sourceTaskManagers[i], jobId);
targetSlots[i] = createSlot(targetTaskManagers[i], jobId);
sourceFutures[i] = new FlinkCompletableFuture<>();
targetFutures[i] = new FlinkCompletableFuture<>();
}
ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
slotProvider.addSlots(sourceVertex.getID(), sourceFutures);
slotProvider.addSlots(targetVertex.getID(), targetFutures);
final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
for (int i = 0; i < parallelism; i += 2) {
sourceFutures[i].complete(sourceSlots[i]);
}
//
// kick off the scheduling
eg.setScheduleMode(ScheduleMode.EAGER);
eg.setQueuedSchedulingAllowed(true);
eg.scheduleForExecution();
verifyNothingDeployed(eg, sourceTaskManagers);
// complete the remaining sources
for (int i = 1; i < parallelism; i += 2) {
sourceFutures[i].complete(sourceSlots[i]);
}
verifyNothingDeployed(eg, sourceTaskManagers);
// complete the targets except for one
for (int i = 1; i < parallelism; i++) {
targetFutures[i].complete(targetSlots[i]);
}
verifyNothingDeployed(eg, targetTaskManagers);
// complete the last target slot future
targetFutures[0].complete(targetSlots[0]);
for (TaskManagerGateway gateway : sourceTaskManagers) {
verify(gateway, timeout(50)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
}
for (TaskManagerGateway gateway : targetTaskManagers) {
verify(gateway, timeout(50)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
}
}
use of org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture in project flink by apache.
the class ExecutionGraphSchedulingTest method testOneSlotFailureAbortsDeploy.
/**
* This test verifies that if one slot future fails, the deployment will be aborted.
*/
@Test
public void testOneSlotFailureAbortsDeploy() throws Exception {
// [pipelined]
// we construct a simple graph (source) ----------------> (target)
final int parallelism = 6;
final JobVertex sourceVertex = new JobVertex("source");
sourceVertex.setParallelism(parallelism);
sourceVertex.setInvokableClass(NoOpInvokable.class);
final JobVertex targetVertex = new JobVertex("target");
targetVertex.setParallelism(parallelism);
targetVertex.setInvokableClass(NoOpInvokable.class);
targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
final JobID jobId = new JobID();
final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
//
// Create the slots, futures, and the slot provider
final TaskManagerGateway taskManager = mock(TaskManagerGateway.class);
final SlotOwner slotOwner = mock(SlotOwner.class);
final SimpleSlot[] sourceSlots = new SimpleSlot[parallelism];
final SimpleSlot[] targetSlots = new SimpleSlot[parallelism];
@SuppressWarnings({ "unchecked", "rawtypes" }) final FlinkCompletableFuture<SimpleSlot>[] sourceFutures = new FlinkCompletableFuture[parallelism];
@SuppressWarnings({ "unchecked", "rawtypes" }) final FlinkCompletableFuture<SimpleSlot>[] targetFutures = new FlinkCompletableFuture[parallelism];
for (int i = 0; i < parallelism; i++) {
sourceSlots[i] = createSlot(taskManager, jobId, slotOwner);
targetSlots[i] = createSlot(taskManager, jobId, slotOwner);
sourceFutures[i] = new FlinkCompletableFuture<>();
targetFutures[i] = new FlinkCompletableFuture<>();
}
ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
slotProvider.addSlots(sourceVertex.getID(), sourceFutures);
slotProvider.addSlots(targetVertex.getID(), targetFutures);
final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
TerminalJobStatusListener testListener = new TerminalJobStatusListener();
eg.registerJobStatusListener(testListener);
for (int i = 0; i < parallelism; i += 2) {
sourceFutures[i].complete(sourceSlots[i]);
targetFutures[i + 1].complete(targetSlots[i + 1]);
}
//
// kick off the scheduling
eg.setScheduleMode(ScheduleMode.EAGER);
eg.setQueuedSchedulingAllowed(true);
eg.scheduleForExecution();
// fail one slot
sourceFutures[1].completeExceptionally(new TestRuntimeException());
// wait until the job failed as a whole
testListener.waitForTerminalState(2000);
// wait until all slots are back
verify(slotOwner, new Timeout(2000, times(6))).returnAllocatedSlot(any(Slot.class));
// no deployment calls must have happened
verify(taskManager, times(0)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
// all completed futures must have been returns
for (int i = 0; i < parallelism; i += 2) {
assertTrue(sourceSlots[i].isCanceled());
assertTrue(targetSlots[i + 1].isCanceled());
}
}
use of org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture in project flink by apache.
the class FlinkFutureTest method testMultipleCompleteOperations.
@Test(timeout = 10000L)
public void testMultipleCompleteOperations() throws ExecutionException, InterruptedException {
CompletableFuture<Integer> initialFuture = new FlinkCompletableFuture<>();
int expectedValue = 42;
assertTrue(initialFuture.complete(expectedValue));
assertFalse(initialFuture.complete(1337));
assertFalse(initialFuture.completeExceptionally(new TestException("foobar")));
assertEquals(new Integer(expectedValue), initialFuture.get());
}
Aggregations