Examples with ConjunctFuture org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture used on opensource projects

Example 1 with ConjunctFuture

use of org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture in project flink by apache.

the class ExecutionGraph method scheduleEager.

/**
	 * 
	 * 
	 * @param slotProvider  The resource provider from which the slots are allocated
	 * @param timeout       The maximum time that the deployment may take, before a
	 *                      TimeoutException is thrown.
	 */
private void scheduleEager(SlotProvider slotProvider, final Time timeout) {
    checkState(state == JobStatus.RUNNING, "job is not running currently");
    // Important: reserve all the space we need up front.
    // that way we do not have any operation that can fail between allocating the slots
    // and adding them to the list. If we had a failure in between there, that would
    // cause the slots to get lost
    final ArrayList<ExecutionAndSlot[]> resources = new ArrayList<>(getNumberOfExecutionJobVertices());
    final boolean queued = allowQueuedScheduling;
    // we use this flag to handle failures in a 'finally' clause
    // that allows us to not go through clumsy cast-and-rethrow logic
    boolean successful = false;
    try {
        // collecting all the slots may resize and fail in that operation without slots getting lost
        final ArrayList<Future<SimpleSlot>> slotFutures = new ArrayList<>(getNumberOfExecutionJobVertices());
        // allocate the slots (obtain all their futures
        for (ExecutionJobVertex ejv : getVerticesTopologically()) {
            // these calls are not blocking, they only return futures
            ExecutionAndSlot[] slots = ejv.allocateResourcesForAll(slotProvider, queued);
            // we need to first add the slots to this list, to be safe on release
            resources.add(slots);
            for (ExecutionAndSlot ens : slots) {
                slotFutures.add(ens.slotFuture);
            }
        }
        // this future is complete once all slot futures are complete.
        // the future fails once one slot future fails.
        final ConjunctFuture allAllocationsComplete = FutureUtils.combineAll(slotFutures);
        // make sure that we fail if the allocation timeout was exceeded
        final ScheduledFuture<?> timeoutCancelHandle = futureExecutor.schedule(new Runnable() {

            @Override
            public void run() {
                // When the timeout triggers, we try to complete the conjunct future with an exception.
                // Note that this is a no-op if the future is already completed
                int numTotal = allAllocationsComplete.getNumFuturesTotal();
                int numComplete = allAllocationsComplete.getNumFuturesCompleted();
                String message = "Could not allocate all requires slots within timeout of " + timeout + ". Slots required: " + numTotal + ", slots allocated: " + numComplete;
                allAllocationsComplete.completeExceptionally(new NoResourceAvailableException(message));
            }
        }, timeout.getSize(), timeout.getUnit());
        allAllocationsComplete.handleAsync(new BiFunction<Void, Throwable, Void>() {

            @Override
            public Void apply(Void ignored, Throwable throwable) {
                try {
                    // we do not need the cancellation timeout any more
                    timeoutCancelHandle.cancel(false);
                    if (throwable == null) {
                        for (ExecutionAndSlot[] jobVertexTasks : resources) {
                            for (ExecutionAndSlot execAndSlot : jobVertexTasks) {
                                // the futures must all be ready - this is simply a sanity check
                                final SimpleSlot slot;
                                try {
                                    slot = execAndSlot.slotFuture.getNow(null);
                                    checkNotNull(slot);
                                } catch (ExecutionException | NullPointerException e) {
                                    throw new IllegalStateException("SlotFuture is incomplete " + "or erroneous even though all futures completed");
                                }
                                // actual deployment
                                execAndSlot.executionAttempt.deployToSlot(slot);
                            }
                        }
                    } else {
                        // let the exception handler deal with this
                        throw throwable;
                    }
                } catch (Throwable t) {
                    // we need to go into recovery and make sure to release all slots
                    try {
                        fail(t);
                    } finally {
                        ExecutionGraphUtils.releaseAllSlotsSilently(resources);
                    }
                }
                // return (Void) Unsafe.getUnsafe().allocateInstance(Void.class);
                return null;
            }
        }, futureExecutor);
        // from now on, slots will be rescued by the the futures and their completion, or by the timeout
        successful = true;
    } finally {
        if (!successful) {
            // we come here only if the 'try' block finished with an exception
            // we release the slots (possibly failing some executions on the way) and
            // let the exception bubble up
            ExecutionGraphUtils.releaseAllSlotsSilently(resources);
        }
    }
}

Example 2 with ConjunctFuture

use of org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture in project flink by apache.

the class FutureUtilsTest method testConjunctFutureCompletion.

@Test
public void testConjunctFutureCompletion() throws Exception {
    // some futures that we combine
    CompletableFuture<Object> future1 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future2 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future3 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future4 = new FlinkCompletableFuture<>();
    // some future is initially completed
    future2.complete(new Object());
    // build the conjunct future
    ConjunctFuture result = FutureUtils.combineAll(Arrays.asList(future1, future2, future3, future4));
    Future<Void> resultMapped = result.thenAccept(new AcceptFunction<Void>() {

        @Override
        public void accept(Void value) {
        }
    });
    assertEquals(4, result.getNumFuturesTotal());
    assertEquals(1, result.getNumFuturesCompleted());
    assertFalse(result.isDone());
    assertFalse(resultMapped.isDone());
    // complete two more futures
    future4.complete(new Object());
    assertEquals(2, result.getNumFuturesCompleted());
    assertFalse(result.isDone());
    assertFalse(resultMapped.isDone());
    future1.complete(new Object());
    assertEquals(3, result.getNumFuturesCompleted());
    assertFalse(result.isDone());
    assertFalse(resultMapped.isDone());
    // complete one future again
    future1.complete(new Object());
    assertEquals(3, result.getNumFuturesCompleted());
    assertFalse(result.isDone());
    assertFalse(resultMapped.isDone());
    // complete the final future
    future3.complete(new Object());
    assertEquals(4, result.getNumFuturesCompleted());
    assertTrue(result.isDone());
    assertTrue(resultMapped.isDone());
}

Example 3 with ConjunctFuture

use of org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture in project flink by apache.

the class FutureUtilsTest method testConjunctFutureFailureOnSuccessive.

@Test
public void testConjunctFutureFailureOnSuccessive() throws Exception {
    CompletableFuture<Object> future1 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future2 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future3 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future4 = new FlinkCompletableFuture<>();
    // build the conjunct future
    ConjunctFuture result = FutureUtils.combineAll(Arrays.asList(future1, future2, future3, future4));
    assertEquals(4, result.getNumFuturesTotal());
    Future<Void> resultMapped = result.thenAccept(new AcceptFunction<Void>() {

        @Override
        public void accept(Void value) {
        }
    });
    future1.complete(new Object());
    future3.complete(new Object());
    future4.complete(new Object());
    future2.completeExceptionally(new IOException());
    assertEquals(3, result.getNumFuturesCompleted());
    assertTrue(result.isDone());
    assertTrue(resultMapped.isDone());
    try {
        result.get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof IOException);
    }
    try {
        resultMapped.get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof IOException);
    }
}

Example 4 with ConjunctFuture

use of org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture in project flink by apache.

the class FutureUtilsTest method testConjunctFutureFailureOnFirst.

@Test
public void testConjunctFutureFailureOnFirst() throws Exception {
    CompletableFuture<Object> future1 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future2 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future3 = new FlinkCompletableFuture<>();
    CompletableFuture<Object> future4 = new FlinkCompletableFuture<>();
    // build the conjunct future
    ConjunctFuture result = FutureUtils.combineAll(Arrays.asList(future1, future2, future3, future4));
    Future<Void> resultMapped = result.thenAccept(new AcceptFunction<Void>() {

        @Override
        public void accept(Void value) {
        }
    });
    assertEquals(4, result.getNumFuturesTotal());
    assertEquals(0, result.getNumFuturesCompleted());
    assertFalse(result.isDone());
    assertFalse(resultMapped.isDone());
    future2.completeExceptionally(new IOException());
    assertEquals(0, result.getNumFuturesCompleted());
    assertTrue(result.isDone());
    assertTrue(resultMapped.isDone());
    try {
        result.get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof IOException);
    }
    try {
        resultMapped.get();
        fail();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof IOException);
    }
}

Example 5 with ConjunctFuture

use of org.apache.flink.runtime.concurrent.FutureUtils.ConjunctFuture in project flink by apache.

the class FutureUtilsTest method testConjunctOfNone.

@Test
public void testConjunctOfNone() throws Exception {
    final ConjunctFuture result = FutureUtils.combineAll(Collections.<Future<Object>>emptyList());
    assertEquals(0, result.getNumFuturesTotal());
    assertEquals(0, result.getNumFuturesCompleted());
    assertTrue(result.isDone());
}