Examples with Execution org.apache.flink.runtime.executiongraph.Execution used on opensource projects

Example 21 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class StackTraceSampleCoordinatorTest method mockExecutionVertex.

// ------------------------------------------------------------------------
private ExecutionVertex mockExecutionVertex(ExecutionAttemptID executionId, ExecutionState state, boolean sendSuccess) {
    Execution exec = mock(Execution.class);
    when(exec.getAttemptId()).thenReturn(executionId);
    when(exec.getState()).thenReturn(state);
    when(exec.requestStackTraceSample(anyInt(), anyInt(), any(Time.class), anyInt(), any(Time.class))).thenReturn(sendSuccess ? FlinkCompletableFuture.completed(mock(StackTraceSampleResponse.class)) : FlinkCompletableFuture.<StackTraceSampleResponse>completedExceptionally(new Exception("Send failed")));
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    when(vertex.getJobvertexId()).thenReturn(new JobVertexID());
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    return vertex;
}

Example 22 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class StackTraceSampleCoordinatorTest method mockExecutionVertexWithTimeout.

private ExecutionVertex mockExecutionVertexWithTimeout(ExecutionAttemptID executionId, ExecutionState state, ScheduledExecutorService scheduledExecutorService, int timeout) {
    final CompletableFuture<StackTraceSampleResponse> future = new FlinkCompletableFuture<>();
    Execution exec = mock(Execution.class);
    when(exec.getAttemptId()).thenReturn(executionId);
    when(exec.getState()).thenReturn(state);
    when(exec.requestStackTraceSample(anyInt(), anyInt(), any(Time.class), anyInt(), any(Time.class))).thenReturn(future);
    scheduledExecutorService.schedule(new Runnable() {

        @Override
        public void run() {
            future.completeExceptionally(new TimeoutException("Timeout"));
        }
    }, timeout, TimeUnit.MILLISECONDS);
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    when(vertex.getJobvertexId()).thenReturn(new JobVertexID());
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    return vertex;
}

Example 23 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class InputChannelDeploymentDescriptor method fromEdges.

// ------------------------------------------------------------------------
/**
	 * Creates an input channel deployment descriptor for each partition.
	 */
public static InputChannelDeploymentDescriptor[] fromEdges(ExecutionEdge[] edges, SimpleSlot consumerSlot, boolean allowLazyDeployment) throws ExecutionGraphException {
    final ResourceID consumerTaskManager = consumerSlot.getTaskManagerID();
    final InputChannelDeploymentDescriptor[] icdd = new InputChannelDeploymentDescriptor[edges.length];
    // Each edge is connected to a different result partition
    for (int i = 0; i < edges.length; i++) {
        final IntermediateResultPartition consumedPartition = edges[i].getSource();
        final Execution producer = consumedPartition.getProducer().getCurrentExecutionAttempt();
        final ExecutionState producerState = producer.getState();
        final SimpleSlot producerSlot = producer.getAssignedResource();
        final ResultPartitionLocation partitionLocation;
        // The producing task needs to be RUNNING or already FINISHED
        if (consumedPartition.isConsumable() && producerSlot != null && (producerState == ExecutionState.RUNNING || producerState == ExecutionState.FINISHED || producerState == ExecutionState.SCHEDULED || producerState == ExecutionState.DEPLOYING)) {
            final TaskManagerLocation partitionTaskManagerLocation = producerSlot.getTaskManagerLocation();
            final ResourceID partitionTaskManager = partitionTaskManagerLocation.getResourceID();
            if (partitionTaskManager.equals(consumerTaskManager)) {
                // Consuming task is deployed to the same TaskManager as the partition => local
                partitionLocation = ResultPartitionLocation.createLocal();
            } else {
                // Different instances => remote
                final ConnectionID connectionId = new ConnectionID(partitionTaskManagerLocation, consumedPartition.getIntermediateResult().getConnectionIndex());
                partitionLocation = ResultPartitionLocation.createRemote(connectionId);
            }
        } else if (allowLazyDeployment) {
            // The producing task might not have registered the partition yet
            partitionLocation = ResultPartitionLocation.createUnknown();
        } else if (producerState == ExecutionState.CANCELING || producerState == ExecutionState.CANCELED || producerState == ExecutionState.FAILED) {
            String msg = "Trying to schedule a task whose inputs were canceled or failed. " + "The producer is in state " + producerState + ".";
            throw new ExecutionGraphException(msg);
        } else {
            String msg = String.format("Trying to eagerly schedule a task whose inputs " + "are not ready (partition consumable? %s, producer state: %s, producer slot: %s).", consumedPartition.isConsumable(), producerState, producerSlot);
            throw new ExecutionGraphException(msg);
        }
        final ResultPartitionID consumedPartitionId = new ResultPartitionID(consumedPartition.getPartitionId(), producer.getAttemptId());
        icdd[i] = new InputChannelDeploymentDescriptor(consumedPartitionId, partitionLocation);
    }
    return icdd;
}

Example 24 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class CheckpointCoordinator method completePendingCheckpoint.

/**
	 * Try to complete the given pending checkpoint.
	 *
	 * Important: This method should only be called in the checkpoint lock scope.
	 *
	 * @param pendingCheckpoint to complete
	 * @throws CheckpointException if the completion failed
	 */
private void completePendingCheckpoint(PendingCheckpoint pendingCheckpoint) throws CheckpointException {
    final long checkpointId = pendingCheckpoint.getCheckpointId();
    CompletedCheckpoint completedCheckpoint = null;
    try {
        // externalize the checkpoint if required
        if (pendingCheckpoint.getProps().externalizeCheckpoint()) {
            completedCheckpoint = pendingCheckpoint.finalizeCheckpointExternalized();
        } else {
            completedCheckpoint = pendingCheckpoint.finalizeCheckpointNonExternalized();
        }
        completedCheckpointStore.addCheckpoint(completedCheckpoint);
        rememberRecentCheckpointId(checkpointId);
        dropSubsumedCheckpoints(checkpointId);
    } catch (Exception exception) {
        // abort the current pending checkpoint if it has not been discarded yet
        if (!pendingCheckpoint.isDiscarded()) {
            pendingCheckpoint.abortError(exception);
        }
        if (completedCheckpoint != null) {
            // we failed to store the completed checkpoint. Let's clean up
            final CompletedCheckpoint cc = completedCheckpoint;
            executor.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        cc.discard();
                    } catch (Throwable t) {
                        LOG.warn("Could not properly discard completed checkpoint {}.", cc.getCheckpointID(), t);
                    }
                }
            });
        }
        throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.', exception);
    } finally {
        pendingCheckpoints.remove(checkpointId);
        triggerQueuedRequests();
    }
    // record the time when this was completed, to calculate
    // the 'min delay between checkpoints'
    lastCheckpointCompletionNanos = System.nanoTime();
    LOG.info("Completed checkpoint {} ({} bytes in {} ms).", checkpointId, completedCheckpoint.getStateSize(), completedCheckpoint.getDuration());
    if (LOG.isDebugEnabled()) {
        StringBuilder builder = new StringBuilder();
        builder.append("Checkpoint state: ");
        for (TaskState state : completedCheckpoint.getTaskStates().values()) {
            builder.append(state);
            builder.append(", ");
        }
        // Remove last two chars ", "
        builder.setLength(builder.length() - 2);
        LOG.debug(builder.toString());
    }
    // send the "notify complete" call to all vertices
    final long timestamp = completedCheckpoint.getTimestamp();
    for (ExecutionVertex ev : tasksToCommitTo) {
        Execution ee = ev.getCurrentExecutionAttempt();
        if (ee != null) {
            ee.notifyCheckpointComplete(checkpointId, timestamp);
        }
    }
}

Example 25 with Execution

use of org.apache.flink.runtime.executiongraph.Execution in project flink by apache.

the class SimpleSlotTest method testSetExecutionVertex.

@Test
public void testSetExecutionVertex() {
    try {
        Execution ev = mock(Execution.class);
        Execution ev_2 = mock(Execution.class);
        // assign to alive slot
        {
            SimpleSlot slot = getSlot();
            assertTrue(slot.setExecutedVertex(ev));
            assertEquals(ev, slot.getExecutedVertex());
            // try to add another one
            assertFalse(slot.setExecutedVertex(ev_2));
            assertEquals(ev, slot.getExecutedVertex());
        }
        // assign to canceled slot
        {
            SimpleSlot slot = getSlot();
            assertTrue(slot.markCancelled());
            assertFalse(slot.setExecutedVertex(ev));
            assertNull(slot.getExecutedVertex());
        }
        // assign to released marked slot
        {
            SimpleSlot slot = getSlot();
            assertTrue(slot.markCancelled());
            assertTrue(slot.markReleased());
            assertFalse(slot.setExecutedVertex(ev));
            assertNull(slot.getExecutedVertex());
        }
        // assign to released
        {
            SimpleSlot slot = getSlot();
            slot.releaseSlot();
            assertFalse(slot.setExecutedVertex(ev));
            assertNull(slot.getExecutedVertex());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}