Examples with ResultPartitionID org.apache.flink.runtime.io.network.partition.ResultPartitionID used on opensource projects

Example 16 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class TaskTest method testOnPartitionStateUpdate.

@Test
public void testOnPartitionStateUpdate() throws Exception {
    IntermediateDataSetID resultId = new IntermediateDataSetID();
    ResultPartitionID partitionId = new ResultPartitionID();
    SingleInputGate inputGate = mock(SingleInputGate.class);
    when(inputGate.getConsumedResultId()).thenReturn(resultId);
    final Task task = createTask(InvokableBlockingInInvoke.class);
    // Set the mock input gate
    setInputGate(task, inputGate);
    // Expected task state for each producer state
    final Map<ExecutionState, ExecutionState> expected = new HashMap<>(ExecutionState.values().length);
    // Fail the task for unexpected states
    for (ExecutionState state : ExecutionState.values()) {
        expected.put(state, ExecutionState.FAILED);
    }
    expected.put(ExecutionState.RUNNING, ExecutionState.RUNNING);
    expected.put(ExecutionState.SCHEDULED, ExecutionState.RUNNING);
    expected.put(ExecutionState.DEPLOYING, ExecutionState.RUNNING);
    expected.put(ExecutionState.FINISHED, ExecutionState.RUNNING);
    expected.put(ExecutionState.CANCELED, ExecutionState.CANCELING);
    expected.put(ExecutionState.CANCELING, ExecutionState.CANCELING);
    expected.put(ExecutionState.FAILED, ExecutionState.CANCELING);
    for (ExecutionState state : ExecutionState.values()) {
        setState(task, ExecutionState.RUNNING);
        task.onPartitionStateUpdate(resultId, partitionId, state);
        ExecutionState newTaskState = task.getExecutionState();
        assertEquals(expected.get(state), newTaskState);
    }
    verify(inputGate, times(4)).retriggerPartitionRequest(eq(partitionId.getPartitionId()));
}

Example 17 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class ExecutionGraphSignalsTest method testFailingScheduleOrUpdateConsumers.

/**
	 * Tests that a failing scheduleOrUpdateConsumers call with a non-existing execution attempt
	 * id, will not fail the execution graph.
	 */
@Test
public void testFailingScheduleOrUpdateConsumers() throws IllegalAccessException {
    IntermediateResultPartitionID intermediateResultPartitionId = new IntermediateResultPartitionID();
    // The execution attempt id does not exist and thus the scheduleOrUpdateConsumers call
    // should fail
    ExecutionAttemptID producerId = new ExecutionAttemptID();
    ResultPartitionID resultPartitionId = new ResultPartitionID(intermediateResultPartitionId, producerId);
    f.set(eg, JobStatus.RUNNING);
    assertEquals(JobStatus.RUNNING, eg.getState());
    try {
        eg.scheduleOrUpdateConsumers(resultPartitionId);
        fail("Expected ExecutionGraphException.");
    } catch (ExecutionGraphException e) {
    // we've expected this exception to occur
    }
    assertEquals(JobStatus.RUNNING, eg.getState());
}

Example 18 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class InputChannelDeploymentDescriptorTest method testMixedLocalRemoteUnknownDeployment.

/**
	 * Tests the deployment descriptors for local, remote, and unknown partition
	 * locations (with lazy deployment allowed and all execution states for the
	 * producers).
	 */
@Test
public void testMixedLocalRemoteUnknownDeployment() throws Exception {
    boolean allowLazyDeployment = true;
    ResourceID consumerResourceId = ResourceID.generate();
    ExecutionVertex consumer = mock(ExecutionVertex.class);
    SimpleSlot consumerSlot = mockSlot(consumerResourceId);
    // states.
    for (ExecutionState state : ExecutionState.values()) {
        // Local partition
        ExecutionVertex localProducer = mockExecutionVertex(state, consumerResourceId);
        IntermediateResultPartition localPartition = mockPartition(localProducer);
        ResultPartitionID localPartitionId = new ResultPartitionID(localPartition.getPartitionId(), localProducer.getCurrentExecutionAttempt().getAttemptId());
        ExecutionEdge localEdge = new ExecutionEdge(localPartition, consumer, 0);
        // Remote partition
        // new resource ID
        ExecutionVertex remoteProducer = mockExecutionVertex(state, ResourceID.generate());
        IntermediateResultPartition remotePartition = mockPartition(remoteProducer);
        ResultPartitionID remotePartitionId = new ResultPartitionID(remotePartition.getPartitionId(), remoteProducer.getCurrentExecutionAttempt().getAttemptId());
        ConnectionID remoteConnectionId = new ConnectionID(remoteProducer.getCurrentAssignedResource().getTaskManagerLocation(), 0);
        ExecutionEdge remoteEdge = new ExecutionEdge(remotePartition, consumer, 1);
        // Unknown partition
        // no assigned resource
        ExecutionVertex unknownProducer = mockExecutionVertex(state, null);
        IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
        ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
        ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
        InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { localEdge, remoteEdge, unknownEdge }, consumerSlot, allowLazyDeployment);
        assertEquals(3, desc.length);
        // These states are allowed
        if (state == ExecutionState.RUNNING || state == ExecutionState.FINISHED || state == ExecutionState.SCHEDULED || state == ExecutionState.DEPLOYING) {
            // Create local or remote channels
            assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
            assertTrue(desc[0].getConsumedPartitionLocation().isLocal());
            assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
            assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
            assertTrue(desc[1].getConsumedPartitionLocation().isRemote());
            assertEquals(remoteConnectionId, desc[1].getConsumedPartitionLocation().getConnectionId());
        } else {
            // Unknown (lazy deployment allowed)
            assertEquals(localPartitionId, desc[0].getConsumedPartitionId());
            assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
            assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
            assertEquals(remotePartitionId, desc[1].getConsumedPartitionId());
            assertTrue(desc[1].getConsumedPartitionLocation().isUnknown());
            assertNull(desc[1].getConsumedPartitionLocation().getConnectionId());
        }
        assertEquals(unknownPartitionId, desc[2].getConsumedPartitionId());
        assertTrue(desc[2].getConsumedPartitionLocation().isUnknown());
        assertNull(desc[2].getConsumedPartitionLocation().getConnectionId());
    }
}

Example 19 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class InputChannelDeploymentDescriptorTest method testUnknownChannelWithoutLazyDeploymentThrows.

@Test
public void testUnknownChannelWithoutLazyDeploymentThrows() throws Exception {
    ResourceID consumerResourceId = ResourceID.generate();
    ExecutionVertex consumer = mock(ExecutionVertex.class);
    SimpleSlot consumerSlot = mockSlot(consumerResourceId);
    // Unknown partition
    // no assigned resource
    ExecutionVertex unknownProducer = mockExecutionVertex(ExecutionState.CREATED, null);
    IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
    ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
    ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
    // This should work if lazy deployment is allowed
    boolean allowLazyDeployment = true;
    InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
    assertEquals(1, desc.length);
    assertEquals(unknownPartitionId, desc[0].getConsumedPartitionId());
    assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
    assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
    try {
        // Fail if lazy deployment is *not* allowed
        allowLazyDeployment = false;
        InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
        fail("Did not throw expected ExecutionGraphException");
    } catch (ExecutionGraphException ignored) {
    }
}

Example 20 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class InputChannelDeploymentDescriptor method fromEdges.

// ------------------------------------------------------------------------
/**
	 * Creates an input channel deployment descriptor for each partition.
	 */
public static InputChannelDeploymentDescriptor[] fromEdges(ExecutionEdge[] edges, SimpleSlot consumerSlot, boolean allowLazyDeployment) throws ExecutionGraphException {
    final ResourceID consumerTaskManager = consumerSlot.getTaskManagerID();
    final InputChannelDeploymentDescriptor[] icdd = new InputChannelDeploymentDescriptor[edges.length];
    // Each edge is connected to a different result partition
    for (int i = 0; i < edges.length; i++) {
        final IntermediateResultPartition consumedPartition = edges[i].getSource();
        final Execution producer = consumedPartition.getProducer().getCurrentExecutionAttempt();
        final ExecutionState producerState = producer.getState();
        final SimpleSlot producerSlot = producer.getAssignedResource();
        final ResultPartitionLocation partitionLocation;
        // The producing task needs to be RUNNING or already FINISHED
        if (consumedPartition.isConsumable() && producerSlot != null && (producerState == ExecutionState.RUNNING || producerState == ExecutionState.FINISHED || producerState == ExecutionState.SCHEDULED || producerState == ExecutionState.DEPLOYING)) {
            final TaskManagerLocation partitionTaskManagerLocation = producerSlot.getTaskManagerLocation();
            final ResourceID partitionTaskManager = partitionTaskManagerLocation.getResourceID();
            if (partitionTaskManager.equals(consumerTaskManager)) {
                // Consuming task is deployed to the same TaskManager as the partition => local
                partitionLocation = ResultPartitionLocation.createLocal();
            } else {
                // Different instances => remote
                final ConnectionID connectionId = new ConnectionID(partitionTaskManagerLocation, consumedPartition.getIntermediateResult().getConnectionIndex());
                partitionLocation = ResultPartitionLocation.createRemote(connectionId);
            }
        } else if (allowLazyDeployment) {
            // The producing task might not have registered the partition yet
            partitionLocation = ResultPartitionLocation.createUnknown();
        } else if (producerState == ExecutionState.CANCELING || producerState == ExecutionState.CANCELED || producerState == ExecutionState.FAILED) {
            String msg = "Trying to schedule a task whose inputs were canceled or failed. " + "The producer is in state " + producerState + ".";
            throw new ExecutionGraphException(msg);
        } else {
            String msg = String.format("Trying to eagerly schedule a task whose inputs " + "are not ready (partition consumable? %s, producer state: %s, producer slot: %s).", consumedPartition.isConsumable(), producerState, producerSlot);
            throw new ExecutionGraphException(msg);
        }
        final ResultPartitionID consumedPartitionId = new ResultPartitionID(consumedPartition.getPartitionId(), producer.getAttemptId());
        icdd[i] = new InputChannelDeploymentDescriptor(consumedPartitionId, partitionLocation);
    }
    return icdd;
}