Search in sources :

Example 1 with SlotProvider

use of org.apache.flink.runtime.instance.SlotProvider in project flink by apache.

the class ExecutionGraphSchedulingTest method testExecutionJobVertexAllocateResourcesReleasesOnException.

/**
	 * Tests that the {@link ExecutionJobVertex#allocateResourcesForAll(SlotProvider, boolean)} method
	 * releases partially acquired resources upon exception.
	 */
@Test
public void testExecutionJobVertexAllocateResourcesReleasesOnException() throws Exception {
    final int parallelism = 8;
    final JobVertex vertex = new JobVertex("vertex");
    vertex.setParallelism(parallelism);
    vertex.setInvokableClass(NoOpInvokable.class);
    final JobID jobId = new JobID();
    final JobGraph jobGraph = new JobGraph(jobId, "test", vertex);
    // set up some available slots and some slot owner that accepts released slots back
    final List<SimpleSlot> returnedSlots = new ArrayList<>();
    final SlotOwner recycler = new SlotOwner() {

        @Override
        public boolean returnAllocatedSlot(Slot slot) {
            returnedSlots.add((SimpleSlot) slot);
            return true;
        }
    };
    // slot provider that hand out parallelism / 3 slots, then throws an exception
    final SlotProvider slotProvider = mock(SlotProvider.class);
    final TaskManagerGateway taskManager = mock(TaskManagerGateway.class);
    final List<SimpleSlot> availableSlots = new ArrayList<>(Arrays.asList(createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler)));
    when(slotProvider.allocateSlot(any(ScheduledUnit.class), anyBoolean())).then(new Answer<Future<SimpleSlot>>() {

        @Override
        public Future<SimpleSlot> answer(InvocationOnMock invocation) {
            if (availableSlots.isEmpty()) {
                throw new TestRuntimeException();
            } else {
                return FlinkCompletableFuture.completed(availableSlots.remove(0));
            }
        }
    });
    final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
    final ExecutionJobVertex ejv = eg.getJobVertex(vertex.getID());
    // acquire resources and check that all are back after the failure
    final int numSlotsToExpectBack = availableSlots.size();
    try {
        ejv.allocateResourcesForAll(slotProvider, false);
        fail("should have failed with an exception");
    } catch (TestRuntimeException e) {
    // expected
    }
    assertEquals(numSlotsToExpectBack, returnedSlots.size());
}
Also used : SlotProvider(org.apache.flink.runtime.instance.SlotProvider) ArrayList(java.util.ArrayList) TaskManagerGateway(org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway) ScheduledUnit(org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit) SimpleSlot(org.apache.flink.runtime.instance.SimpleSlot) SlotOwner(org.apache.flink.runtime.jobmanager.slots.SlotOwner) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Slot(org.apache.flink.runtime.instance.Slot) SimpleSlot(org.apache.flink.runtime.instance.SimpleSlot) AllocatedSlot(org.apache.flink.runtime.jobmanager.slots.AllocatedSlot) Future(org.apache.flink.runtime.concurrent.Future) FlinkCompletableFuture(org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Example 2 with SlotProvider

use of org.apache.flink.runtime.instance.SlotProvider in project flink by apache.

the class ExecutionGraphSchedulingTest method testExecutionGraphScheduleReleasesResourcesOnException.

/**
	 * Tests that the {@link ExecutionGraph#scheduleForExecution()} method
	 * releases partially acquired resources upon exception.
	 */
@Test
public void testExecutionGraphScheduleReleasesResourcesOnException() throws Exception {
    //                                            [pipelined]
    //  we construct a simple graph    (source) ----------------> (target)
    final int parallelism = 3;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setParallelism(parallelism);
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    final JobVertex targetVertex = new JobVertex("target");
    targetVertex.setParallelism(parallelism);
    targetVertex.setInvokableClass(NoOpInvokable.class);
    targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobID jobId = new JobID();
    final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
    // set up some available slots and some slot owner that accepts released slots back
    final List<SimpleSlot> returnedSlots = new ArrayList<>();
    final SlotOwner recycler = new SlotOwner() {

        @Override
        public boolean returnAllocatedSlot(Slot slot) {
            returnedSlots.add((SimpleSlot) slot);
            return true;
        }
    };
    final TaskManagerGateway taskManager = mock(TaskManagerGateway.class);
    final List<SimpleSlot> availableSlots = new ArrayList<>(Arrays.asList(createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler)));
    // slot provider that hand out parallelism / 3 slots, then throws an exception
    final SlotProvider slotProvider = mock(SlotProvider.class);
    when(slotProvider.allocateSlot(any(ScheduledUnit.class), anyBoolean())).then(new Answer<Future<SimpleSlot>>() {

        @Override
        public Future<SimpleSlot> answer(InvocationOnMock invocation) {
            if (availableSlots.isEmpty()) {
                throw new TestRuntimeException();
            } else {
                return FlinkCompletableFuture.completed(availableSlots.remove(0));
            }
        }
    });
    final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
    // acquire resources and check that all are back after the failure
    final int numSlotsToExpectBack = availableSlots.size();
    try {
        eg.setScheduleMode(ScheduleMode.EAGER);
        eg.scheduleForExecution();
        fail("should have failed with an exception");
    } catch (TestRuntimeException e) {
    // expected
    }
    assertEquals(numSlotsToExpectBack, returnedSlots.size());
}
Also used : SlotProvider(org.apache.flink.runtime.instance.SlotProvider) ArrayList(java.util.ArrayList) TaskManagerGateway(org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway) ScheduledUnit(org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit) SimpleSlot(org.apache.flink.runtime.instance.SimpleSlot) SlotOwner(org.apache.flink.runtime.jobmanager.slots.SlotOwner) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Slot(org.apache.flink.runtime.instance.Slot) SimpleSlot(org.apache.flink.runtime.instance.SimpleSlot) AllocatedSlot(org.apache.flink.runtime.jobmanager.slots.AllocatedSlot) Future(org.apache.flink.runtime.concurrent.Future) FlinkCompletableFuture(org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)2 JobID (org.apache.flink.api.common.JobID)2 Future (org.apache.flink.runtime.concurrent.Future)2 FlinkCompletableFuture (org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture)2 SimpleSlot (org.apache.flink.runtime.instance.SimpleSlot)2 Slot (org.apache.flink.runtime.instance.Slot)2 SlotProvider (org.apache.flink.runtime.instance.SlotProvider)2 JobGraph (org.apache.flink.runtime.jobgraph.JobGraph)2 JobVertex (org.apache.flink.runtime.jobgraph.JobVertex)2 ScheduledUnit (org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit)2 AllocatedSlot (org.apache.flink.runtime.jobmanager.slots.AllocatedSlot)2 SlotOwner (org.apache.flink.runtime.jobmanager.slots.SlotOwner)2 TaskManagerGateway (org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway)2 Test (org.junit.Test)2 InvocationOnMock (org.mockito.invocation.InvocationOnMock)2