Examples with ExecutionVertex org.apache.flink.runtime.executiongraph.ExecutionVertex used on opensource projects

Example 26 with ExecutionVertex

use of org.apache.flink.runtime.executiongraph.ExecutionVertex in project flink by apache.

the class Scheduler method handleNewSlot.

private void handleNewSlot() {
    synchronized (globalLock) {
        Instance instance = this.newlyAvailableInstances.poll();
        if (instance == null || !instance.hasResourcesAvailable()) {
            // someone else took it
            return;
        }
        QueuedTask queued = taskQueue.peek();
        if (queued != null) {
            ScheduledUnit task = queued.getTask();
            ExecutionVertex vertex = task.getTaskToExecute().getVertex();
            try {
                SimpleSlot newSlot = instance.allocateSimpleSlot(vertex.getJobId());
                if (newSlot != null) {
                    // success, remove from the task queue and notify the future
                    taskQueue.poll();
                    if (queued.getFuture() != null) {
                        try {
                            queued.getFuture().complete(newSlot);
                        } catch (Throwable t) {
                            LOG.error("Error calling allocation future for task " + vertex.getSimpleName(), t);
                            task.getTaskToExecute().fail(t);
                        }
                    }
                }
            } catch (InstanceDiedException e) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Instance " + instance + " was marked dead asynchronously.");
                }
                removeInstance(instance);
            }
        } else {
            this.instancesWithAvailableResources.put(instance.getTaskManagerID(), instance);
        }
    }
}

Example 27 with ExecutionVertex

use of org.apache.flink.runtime.executiongraph.ExecutionVertex in project flink by apache.

the class InputChannelDeploymentDescriptorTest method testMixedLocalRemoteUnknownDeployment.

/**
	 * Tests the deployment descriptors for local, remote, and unknown partition
	 * locations (with lazy deployment allowed and all execution states for the
	 * producers).
	 */
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
    boolean allowLazyDeployment = true;
    ResourceID consumerResourceId = ResourceID.generate();
    ExecutionVertex consumer = mock(ExecutionVertex.class);
    SimpleSlot consumerSlot = mockSlot(consumerResourceId);
    // states.
    for (ExecutionState state : ExecutionState.values()) {
        // Local partition
        ExecutionVertex localProducer = mockExecutionVertex(state, consumerResourceId);
        IntermediateResultPartition localPartition = mockPartition(localProducer);
        ResultPartitionID localPartitionId = new ResultPartitionID(localPartition.getPartitionId(), localProducer.getCurrentExecutionAttempt().getAttemptId());
        ExecutionEdge localEdge = new ExecutionEdge(localPartition, consumer, 0);
        // Remote partition
        // new resource ID
        ExecutionVertex remoteProducer = mockExecutionVertex(state, ResourceID.generate());
        IntermediateResultPartition remotePartition = mockPartition(remoteProducer);
        ResultPartitionID remotePartitionId = new ResultPartitionID(remotePartition.getPartitionId(), remoteProducer.getCurrentExecutionAttempt().getAttemptId());
        ConnectionID remoteConnectionId = new ConnectionID(remoteProducer.getCurrentAssignedResource().getTaskManagerLocation(), 0);
        ExecutionEdge remoteEdge = new ExecutionEdge(remotePartition, consumer, 1);
        // Unknown partition
        // no assigned resource
        ExecutionVertex unknownProducer = mockExecutionVertex(state, null);
        IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
        ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
        ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
        InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { localEdge, remoteEdge, unknownEdge }, consumerSlot, allowLazyDeployment);
        assertEquals(3, desc.length);
        // These states are allowed
        if (state == ExecutionState.RUNNING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
            // Create local or remote channels
            assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
            assertTrue(desc[0].getConsumedPartitionLocation().isLocal());
            assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
            assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
            assertTrue(desc[1].getConsumedPartitionLocation().isRemote());
            assertEquals(remoteConnectionId, desc[1].getConsumedPartitionLocation().getConnectionId());
        } else {
            // Unknown (lazy deployment allowed)
            assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
            assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
            assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
            assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
            assertTrue(desc[1].getConsumedPartitionLocation().isUnknown());
            assertNull(desc[1].getConsumedPartitionLocation().getConnectionId());
        }
        assertEquals(unknownPartitionId, desc[2].getConsumedPartitionId());
        assertTrue(desc[2].getConsumedPartitionLocation().isUnknown());
        assertNull(desc[2].getConsumedPartitionLocation().getConnectionId());
    }
}

Example 28 with ExecutionVertex

use of org.apache.flink.runtime.executiongraph.ExecutionVertex in project flink by apache.

the class InputChannelDeploymentDescriptorTest method testUnknownChannelWithoutLazyDeploymentThrows.

@Test
public void testUnknownChannelWithoutLazyDeploymentThrows() throws Exception {
    ResourceID consumerResourceId = ResourceID.generate();
    ExecutionVertex consumer = mock(ExecutionVertex.class);
    SimpleSlot consumerSlot = mockSlot(consumerResourceId);
    // Unknown partition
    // no assigned resource
    ExecutionVertex unknownProducer = mockExecutionVertex(ExecutionState.CREATED, null);
    IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
    ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
    ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
    // This should work if lazy deployment is allowed
    boolean allowLazyDeployment = true;
    InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
    assertEquals(1, desc.length);
    assertEquals(unknownPartitionId, desc[0].getConsumedPartitionId());
    assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
    assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
    try {
        // Fail if lazy deployment is *not* allowed
        allowLazyDeployment = false;
        InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
        fail("Did not throw expected ExecutionGraphException");
    } catch (ExecutionGraphException ignored) {
    }
}

Example 29 with ExecutionVertex

use of org.apache.flink.runtime.executiongraph.ExecutionVertex in project flink by apache.

the class InputChannelDeploymentDescriptorTest method mockExecutionVertex.

private static ExecutionVertex mockExecutionVertex(ExecutionState state, ResourceID resourceId) {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    Execution exec = mock(Execution.class);
    when(exec.getState()).thenReturn(state);
    when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());
    if (resourceId != null) {
        SimpleSlot slot = mockSlot(resourceId);
        when(exec.getAssignedResource()).thenReturn(slot);
        when(vertex.getCurrentAssignedResource()).thenReturn(slot);
    } else {
        // no resource
        when(exec.getAssignedResource()).thenReturn(null);
        when(vertex.getCurrentAssignedResource()).thenReturn(null);
    }
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    return vertex;
}

Example 30 with ExecutionVertex

use of org.apache.flink.runtime.executiongraph.ExecutionVertex in project flink by apache.

the class BackPressureStatsTrackerTest method mockExecutionVertex.

private ExecutionVertex mockExecutionVertex(ExecutionJobVertex jobVertex, int subTaskIndex) {
    Execution exec = mock(Execution.class);
    when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());
    JobVertexID id = jobVertex.getJobVertexId();
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    when(vertex.getJobvertexId()).thenReturn(id);
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    when(vertex.getParallelSubtaskIndex()).thenReturn(subTaskIndex);
    return vertex;
}