Examples with ResultPartitionID org.apache.flink.runtime.io.network.partition.ResultPartitionID used on opensource projects

Example 21 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class PartialInputChannelDeploymentDescriptor method fromEdge.

// ------------------------------------------------------------------------
/**
	 * Creates a partial input channel for the given partition and producing task.
	 */
public static PartialInputChannelDeploymentDescriptor fromEdge(IntermediateResultPartition partition, Execution producer) {
    final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), producer.getAttemptId());
    final IntermediateResult result = partition.getIntermediateResult();
    final IntermediateDataSetID resultId = result.getId();
    final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation();
    final int partitionConnectionIndex = result.getConnectionIndex();
    return new PartialInputChannelDeploymentDescriptor(resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex);
}

Example 22 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class Execution method scheduleOrUpdateConsumers.

void scheduleOrUpdateConsumers(List<List<ExecutionEdge>> allConsumers) {
    final int numConsumers = allConsumers.size();
    if (numConsumers > 1) {
        fail(new IllegalStateException("Currently, only a single consumer group per partition is supported."));
    } else if (numConsumers == 0) {
        return;
    }
    for (ExecutionEdge edge : allConsumers.get(0)) {
        final ExecutionVertex consumerVertex = edge.getTarget();
        final Execution consumer = consumerVertex.getCurrentExecutionAttempt();
        final ExecutionState consumerState = consumer.getState();
        final IntermediateResultPartition partition = edge.getSource();
        // ----------------------------------------------------------------
        if (consumerState == CREATED) {
            final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();
            consumerVertex.cachePartitionInfo(PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));
            // When deploying a consuming task, its task deployment descriptor will contain all
            // deployment information available at the respective time. It is possible that some
            // of the partitions to be consumed have not been created yet. These are updated
            // runtime via the update messages.
            //
            // TODO The current approach may send many update messages even though the consuming
            // task has already been deployed with all necessary information. We have to check
            // whether this is a problem and fix it, if it is.
            FlinkFuture.supplyAsync(new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        consumerVertex.scheduleForExecution(consumerVertex.getExecutionGraph().getSlotProvider(), consumerVertex.getExecutionGraph().isQueuedSchedulingAllowed());
                    } catch (Throwable t) {
                        consumerVertex.fail(new IllegalStateException("Could not schedule consumer " + "vertex " + consumerVertex, t));
                    }
                    return null;
                }
            }, executor);
            // double check to resolve race conditions
            if (consumerVertex.getExecutionState() == RUNNING) {
                consumerVertex.sendPartitionInfos();
            }
        } else // ----------------------------------------------------------------
        // Consumer is running => send update message now
        // ----------------------------------------------------------------
        {
            if (consumerState == RUNNING) {
                final SimpleSlot consumerSlot = consumer.getAssignedResource();
                if (consumerSlot == null) {
                    // The consumer has been reset concurrently
                    continue;
                }
                final TaskManagerLocation partitionTaskManagerLocation = partition.getProducer().getCurrentAssignedResource().getTaskManagerLocation();
                final ResourceID partitionTaskManager = partitionTaskManagerLocation.getResourceID();
                final ResourceID consumerTaskManager = consumerSlot.getTaskManagerID();
                final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), attemptId);
                final ResultPartitionLocation partitionLocation;
                if (consumerTaskManager.equals(partitionTaskManager)) {
                    // Consuming task is deployed to the same instance as the partition => local
                    partitionLocation = ResultPartitionLocation.createLocal();
                } else {
                    // Different instances => remote
                    final ConnectionID connectionId = new ConnectionID(partitionTaskManagerLocation, partition.getIntermediateResult().getConnectionIndex());
                    partitionLocation = ResultPartitionLocation.createRemote(connectionId);
                }
                final InputChannelDeploymentDescriptor descriptor = new InputChannelDeploymentDescriptor(partitionId, partitionLocation);
                consumer.sendUpdatePartitionInfoRpcCall(Collections.singleton(new PartitionInfo(partition.getIntermediateResult().getId(), descriptor)));
            } else // ----------------------------------------------------------------
            if (consumerState == SCHEDULED || consumerState == DEPLOYING) {
                final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();
                consumerVertex.cachePartitionInfo(PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));
                // double check to resolve race conditions
                if (consumerVertex.getExecutionState() == RUNNING) {
                    consumerVertex.sendPartitionInfos();
                }
            }
        }
    }
}

Example 23 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class SingleInputGate method create.

// ------------------------------------------------------------------------
/**
	 * Creates an input gate and all of its input channels.
	 */
public static SingleInputGate create(String owningTaskName, JobID jobId, ExecutionAttemptID executionId, InputGateDeploymentDescriptor igdd, NetworkEnvironment networkEnvironment, TaskActions taskActions, TaskIOMetricGroup metrics) {
    final IntermediateDataSetID consumedResultId = checkNotNull(igdd.getConsumedResultId());
    final ResultPartitionType consumedPartitionType = checkNotNull(igdd.getConsumedPartitionType());
    final int consumedSubpartitionIndex = igdd.getConsumedSubpartitionIndex();
    checkArgument(consumedSubpartitionIndex >= 0);
    final InputChannelDeploymentDescriptor[] icdd = checkNotNull(igdd.getInputChannelDeploymentDescriptors());
    final SingleInputGate inputGate = new SingleInputGate(owningTaskName, jobId, consumedResultId, consumedPartitionType, consumedSubpartitionIndex, icdd.length, taskActions, metrics);
    // Create the input channels. There is one input channel for each consumed partition.
    final InputChannel[] inputChannels = new InputChannel[icdd.length];
    int numLocalChannels = 0;
    int numRemoteChannels = 0;
    int numUnknownChannels = 0;
    for (int i = 0; i < inputChannels.length; i++) {
        final ResultPartitionID partitionId = icdd[i].getConsumedPartitionId();
        final ResultPartitionLocation partitionLocation = icdd[i].getConsumedPartitionLocation();
        if (partitionLocation.isLocal()) {
            inputChannels[i] = new LocalInputChannel(inputGate, i, partitionId, networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics);
            numLocalChannels++;
        } else if (partitionLocation.isRemote()) {
            inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId, partitionLocation.getConnectionId(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics);
            numRemoteChannels++;
        } else if (partitionLocation.isUnknown()) {
            inputChannels[i] = new UnknownInputChannel(inputGate, i, partitionId, networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics);
            numUnknownChannels++;
        } else {
            throw new IllegalStateException("Unexpected partition location.");
        }
        inputGate.setInputChannel(partitionId.getPartitionId(), inputChannels[i]);
    }
    LOG.debug("Created {} input channels (local: {}, remote: {}, unknown: {}).", inputChannels.length, numLocalChannels, numRemoteChannels, numUnknownChannels);
    return inputGate;
}

Example 24 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class ResultPartitionWriterTest method testWriteBufferToAllChannelsReferenceCounting.

// ---------------------------------------------------------------------------------------------
// Resource release tests
// ---------------------------------------------------------------------------------------------
/**
	 * Tests that event buffers are properly recycled when broadcasting events
	 * to multiple channels.
	 *
	 * @throws Exception
	 */
@Test
public void testWriteBufferToAllChannelsReferenceCounting() throws Exception {
    Buffer buffer = EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE);
    ResultPartition partition = new ResultPartition("TestTask", mock(TaskActions.class), new JobID(), new ResultPartitionID(), ResultPartitionType.PIPELINED, 2, 2, mock(ResultPartitionManager.class), mock(ResultPartitionConsumableNotifier.class), mock(IOManager.class), false);
    ResultPartitionWriter partitionWriter = new ResultPartitionWriter(partition);
    partitionWriter.writeBufferToAllChannels(buffer);
    // Verify added to all queues, i.e. two buffers in total
    assertEquals(2, partition.getTotalNumberOfBuffers());
    // release the buffers in the partition
    partition.release();
    assertTrue(buffer.isRecycled());
}

Example 25 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class CancelPartitionRequestTest method testCancelPartitionRequest.

/**
	 * Verifies that requests for non-existing (failed/cancelled) input channels are properly
	 * cancelled. The receiver receives data, but there is no input channel to receive the data.
	 * This should cancel the request.
	 */
@Test
public void testCancelPartitionRequest() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
                listener.notifyBuffersAvailable(Long.MAX_VALUE);
                return view;
            }
        });
        PartitionRequestProtocol protocol = new PartitionRequestProtocol(partitions, mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        ch.writeAndFlush(new PartitionRequest(pid, 0, new InputChannelID())).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() + " ms to be notified about cancelled partition.");
        }
        verify(view, times(1)).releaseAllResources();
        verify(view, times(0)).notifySubpartitionConsumed();
    } finally {
        shutdown(serverAndClient);
    }
}