Examples with Future org.apache.flink.runtime.concurrent.Future used on opensource projects

Example 1 with Future

use of org.apache.flink.runtime.concurrent.Future in project flink by apache.

the class ProgrammedSlotProvider method allocateSlot.

@Override
public Future<SimpleSlot> allocateSlot(ScheduledUnit task, boolean allowQueued) {
    JobVertexID vertexId = task.getTaskToExecute().getVertex().getJobvertexId();
    int subtask = task.getTaskToExecute().getParallelSubtaskIndex();
    Future<SimpleSlot>[] forTask = slotFutures.get(vertexId);
    if (forTask != null) {
        Future<SimpleSlot> future = forTask[subtask];
        if (future != null) {
            return future;
        }
    }
    throw new IllegalArgumentException("No registered slot future for task " + vertexId + " (" + subtask + ')');
}

Example 2 with Future

use of org.apache.flink.runtime.concurrent.Future in project flink by apache.

the class ExecutionGraphSchedulingTest method testExecutionJobVertexAllocateResourcesReleasesOnException.

/**
	 * Tests that the {@link ExecutionJobVertex#allocateResourcesForAll(SlotProvider, boolean)} method
	 * releases partially acquired resources upon exception.
	 */
@Test
public void testExecutionJobVertexAllocateResourcesReleasesOnException() throws Exception {
    final int parallelism = 8;
    final JobVertex vertex = new JobVertex("vertex");
    vertex.setParallelism(parallelism);
    vertex.setInvokableClass(NoOpInvokable.class);
    final JobID jobId = new JobID();
    final JobGraph jobGraph = new JobGraph(jobId, "test", vertex);
    // set up some available slots and some slot owner that accepts released slots back
    final List<SimpleSlot> returnedSlots = new ArrayList<>();
    final SlotOwner recycler = new SlotOwner() {

        @Override
        public boolean returnAllocatedSlot(Slot slot) {
            returnedSlots.add((SimpleSlot) slot);
            return true;
        }
    };
    // slot provider that hand out parallelism / 3 slots, then throws an exception
    final SlotProvider slotProvider = mock(SlotProvider.class);
    final TaskManagerGateway taskManager = mock(TaskManagerGateway.class);
    final List<SimpleSlot> availableSlots = new ArrayList<>(Arrays.asList(createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler)));
    when(slotProvider.allocateSlot(any(ScheduledUnit.class), anyBoolean())).then(new Answer<Future<SimpleSlot>>() {

        @Override
        public Future<SimpleSlot> answer(InvocationOnMock invocation) {
            if (availableSlots.isEmpty()) {
                throw new TestRuntimeException();
            } else {
                return FlinkCompletableFuture.completed(availableSlots.remove(0));
            }
        }
    });
    final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
    final ExecutionJobVertex ejv = eg.getJobVertex(vertex.getID());
    // acquire resources and check that all are back after the failure
    final int numSlotsToExpectBack = availableSlots.size();
    try {
        ejv.allocateResourcesForAll(slotProvider, false);
        fail("should have failed with an exception");
    } catch (TestRuntimeException e) {
    // expected
    }
    assertEquals(numSlotsToExpectBack, returnedSlots.size());
}

Example 3 with Future

use of org.apache.flink.runtime.concurrent.Future in project flink by apache.

the class ExecutionGraphSchedulingTest method testExecutionGraphScheduleReleasesResourcesOnException.

/**
	 * Tests that the {@link ExecutionGraph#scheduleForExecution()} method
	 * releases partially acquired resources upon exception.
	 */
@Test
public void testExecutionGraphScheduleReleasesResourcesOnException() throws Exception {
    //                                            [pipelined]
    //  we construct a simple graph    (source) ----------------> (target)
    final int parallelism = 3;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setParallelism(parallelism);
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    final JobVertex targetVertex = new JobVertex("target");
    targetVertex.setParallelism(parallelism);
    targetVertex.setInvokableClass(NoOpInvokable.class);
    targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobID jobId = new JobID();
    final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
    // set up some available slots and some slot owner that accepts released slots back
    final List<SimpleSlot> returnedSlots = new ArrayList<>();
    final SlotOwner recycler = new SlotOwner() {

        @Override
        public boolean returnAllocatedSlot(Slot slot) {
            returnedSlots.add((SimpleSlot) slot);
            return true;
        }
    };
    final TaskManagerGateway taskManager = mock(TaskManagerGateway.class);
    final List<SimpleSlot> availableSlots = new ArrayList<>(Arrays.asList(createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler), createSlot(taskManager, jobId, recycler)));
    // slot provider that hand out parallelism / 3 slots, then throws an exception
    final SlotProvider slotProvider = mock(SlotProvider.class);
    when(slotProvider.allocateSlot(any(ScheduledUnit.class), anyBoolean())).then(new Answer<Future<SimpleSlot>>() {

        @Override
        public Future<SimpleSlot> answer(InvocationOnMock invocation) {
            if (availableSlots.isEmpty()) {
                throw new TestRuntimeException();
            } else {
                return FlinkCompletableFuture.completed(availableSlots.remove(0));
            }
        }
    });
    final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
    // acquire resources and check that all are back after the failure
    final int numSlotsToExpectBack = availableSlots.size();
    try {
        eg.setScheduleMode(ScheduleMode.EAGER);
        eg.scheduleForExecution();
        fail("should have failed with an exception");
    } catch (TestRuntimeException e) {
    // expected
    }
    assertEquals(numSlotsToExpectBack, returnedSlots.size());
}

Example 4 with Future

use of org.apache.flink.runtime.concurrent.Future in project flink by apache.

the class AkkaInvocationHandler method invoke.

@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
    Class<?> declaringClass = method.getDeclaringClass();
    Object result;
    if (declaringClass.equals(AkkaGateway.class) || declaringClass.equals(MainThreadExecutable.class) || declaringClass.equals(Object.class) || declaringClass.equals(StartStoppable.class) || declaringClass.equals(RpcGateway.class) || declaringClass.equals(SelfGateway.class)) {
        result = method.invoke(this, args);
    } else {
        String methodName = method.getName();
        Class<?>[] parameterTypes = method.getParameterTypes();
        Annotation[][] parameterAnnotations = method.getParameterAnnotations();
        Time futureTimeout = extractRpcTimeout(parameterAnnotations, args, timeout);
        Tuple2<Class<?>[], Object[]> filteredArguments = filterArguments(parameterTypes, parameterAnnotations, args);
        RpcInvocation rpcInvocation;
        if (isLocal) {
            rpcInvocation = new LocalRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
        } else {
            try {
                RemoteRpcInvocation remoteRpcInvocation = new RemoteRpcInvocation(methodName, filteredArguments.f0, filteredArguments.f1);
                if (remoteRpcInvocation.getSize() > maximumFramesize) {
                    throw new IOException("The rpc invocation size exceeds the maximum akka framesize.");
                } else {
                    rpcInvocation = remoteRpcInvocation;
                }
            } catch (IOException e) {
                LOG.warn("Could not create remote rpc invocation message. Failing rpc invocation because...", e);
                throw e;
            }
        }
        Class<?> returnType = method.getReturnType();
        if (returnType.equals(Void.TYPE)) {
            rpcEndpoint.tell(rpcInvocation, ActorRef.noSender());
            result = null;
        } else if (returnType.equals(Future.class)) {
            // execute an asynchronous call
            result = new FlinkFuture<>(Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds()));
        } else {
            // execute a synchronous call
            scala.concurrent.Future<?> scalaFuture = Patterns.ask(rpcEndpoint, rpcInvocation, futureTimeout.toMilliseconds());
            Future<?> futureResult = new FlinkFuture<>(scalaFuture);
            return futureResult.get(futureTimeout.getSize(), futureTimeout.getUnit());
        }
    }
    return result;
}

Example 5 with Future

use of org.apache.flink.runtime.concurrent.Future in project flink by apache.

the class Execution method stop.

/**
	 * Sends stop RPC call.
	 */
public void stop() {
    final SimpleSlot slot = assignedResource;
    if (slot != null) {
        final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
        Future<Acknowledge> stopResultFuture = FutureUtils.retry(new Callable<Future<Acknowledge>>() {

            @Override
            public Future<Acknowledge> call() throws Exception {
                return taskManagerGateway.stopTask(attemptId, timeout);
            }
        }, NUM_STOP_CALL_TRIES, executor);
        stopResultFuture.exceptionally(new ApplyFunction<Throwable, Void>() {

            @Override
            public Void apply(Throwable failure) {
                LOG.info("Stopping task was not successful.", failure);
                return null;
            }
        });
    }
}