Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 1 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironment method unregisterTask.

public void unregisterTask(Task task) {
    LOG.debug("Unregister task {} from network environment (state: {}).", task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());
    final ExecutionAttemptID executionId = task.getExecutionId();
    synchronized (lock) {
        if (isShutdown) {
            // no need to do anything when we are not operational
            return;
        }
        if (task.isCanceledOrFailed()) {
            resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
        }
        ResultPartitionWriter[] writers = task.getAllWriters();
        if (writers != null) {
            for (ResultPartitionWriter writer : writers) {
                taskEventDispatcher.unregisterWriter(writer);
            }
        }
        ResultPartition[] partitions = task.getProducedPartitions();
        if (partitions != null) {
            for (ResultPartition partition : partitions) {
                partition.destroyBufferPool();
            }
        }
        final SingleInputGate[] inputGates = task.getAllInputGates();
        if (inputGates != null) {
            for (SingleInputGate gate : inputGates) {
                try {
                    if (gate != null) {
                        gate.releaseAllResources();
                    }
                } catch (IOException e) {
                    LOG.error("Error during release of reader resources: " + e.getMessage(), e);
                }
            }
        }
    }
}

Example 2 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithLocalChannels.

@Test
public void testConsumptionWithLocalChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final PipelinedSubpartition[] partitions = new PipelinedSubpartition[numChannels];
    final Source[] sources = new Source[numChannels];
    final ResultPartitionManager resultPartitionManager = createResultPartitionManager(partitions);
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        partitions[i] = new PipelinedSubpartition(0, resultPartition);
        sources[i] = new PipelinedSubpartitionSource(partitions[i]);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 3 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithRemoteChannels.

@Test
public void testConsumptionWithRemoteChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ConnectionManager connManager = createDummyConnectionManager();
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        sources[i] = new RemoteChannelSource(channel);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 4 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironmentTest method testRegisterTaskUsesBoundedBuffers.

/**
	 * Verifies that {@link NetworkEnvironment#registerTask(Task)} sets up (un)bounded buffer pool
	 * instances for various types of input and output channels.
	 */
@Test
public void testRegisterTaskUsesBoundedBuffers() throws Exception {
    final NetworkEnvironment network = new NetworkEnvironment(new NetworkBufferPool(numBuffers, memorySegmentSize, MemoryType.HEAP), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, IOManager.IOMode.SYNC, 0, 0, 2, 8);
    // result partitions
    ResultPartition rp1 = createResultPartition(ResultPartitionType.PIPELINED, 2);
    ResultPartition rp2 = createResultPartition(ResultPartitionType.BLOCKING, 2);
    ResultPartition rp3 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 2);
    ResultPartition rp4 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 8);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    final ResultPartitionWriter[] resultPartitionWriters = new ResultPartitionWriter[] { new ResultPartitionWriter(rp1), new ResultPartitionWriter(rp2), new ResultPartitionWriter(rp3), new ResultPartitionWriter(rp4) };
    // input gates
    final SingleInputGate[] inputGates = new SingleInputGate[] { createSingleInputGateMock(ResultPartitionType.PIPELINED, 2), createSingleInputGateMock(ResultPartitionType.BLOCKING, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 8) };
    // overall task to register
    Task task = mock(Task.class);
    when(task.getProducedPartitions()).thenReturn(resultPartitions);
    when(task.getAllWriters()).thenReturn(resultPartitionWriters);
    when(task.getAllInputGates()).thenReturn(inputGates);
    network.registerTask(task);
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(2 * 2 + 8, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(8 * 2 + 8, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    network.shutdown();
}

Example 5 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironmentTest method createSingleInputGateMock.

/**
	 * Helper to create a mock of a {@link SingleInputGate} for use by a {@link Task} inside
	 * {@link NetworkEnvironment#registerTask(Task)}.
	 *
	 * @param partitionType
	 * 		the consumed partition type
	 * @param channels
	 * 		the nummer of input channels
	 *
	 * @return mock with minimal functionality necessary by {@link NetworkEnvironment#registerTask(Task)}
	 */
private static SingleInputGate createSingleInputGateMock(final ResultPartitionType partitionType, final int channels) {
    SingleInputGate ig = mock(SingleInputGate.class);
    when(ig.getConsumedPartitionType()).thenReturn(partitionType);
    when(ig.getNumberOfInputChannels()).thenReturn(channels);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(final InvocationOnMock invocation) throws Throwable {
            BufferPool bp = invocation.getArgumentAt(0, BufferPool.class);
            if (partitionType == ResultPartitionType.PIPELINED_BOUNDED) {
                assertEquals(channels * 2 + 8, bp.getMaxNumberOfMemorySegments());
            } else {
                assertEquals(Integer.MAX_VALUE, bp.getMaxNumberOfMemorySegments());
            }
            return null;
        }
    }).when(ig).setBufferPool(any(BufferPool.class));
    return ig;
}