Search in sources :

Example 1 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironment method unregisterTask.

public void unregisterTask(Task task) {
    LOG.debug("Unregister task {} from network environment (state: {}).", task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());
    final ExecutionAttemptID executionId = task.getExecutionId();
    synchronized (lock) {
        if (isShutdown) {
            // no need to do anything when we are not operational
            return;
        }
        if (task.isCanceledOrFailed()) {
            resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
        }
        ResultPartitionWriter[] writers = task.getAllWriters();
        if (writers != null) {
            for (ResultPartitionWriter writer : writers) {
                taskEventDispatcher.unregisterWriter(writer);
            }
        }
        ResultPartition[] partitions = task.getProducedPartitions();
        if (partitions != null) {
            for (ResultPartition partition : partitions) {
                partition.destroyBufferPool();
            }
        }
        final SingleInputGate[] inputGates = task.getAllInputGates();
        if (inputGates != null) {
            for (SingleInputGate gate : inputGates) {
                try {
                    if (gate != null) {
                        gate.releaseAllResources();
                    }
                } catch (IOException e) {
                    LOG.error("Error during release of reader resources: " + e.getMessage(), e);
                }
            }
        }
    }
}
Also used : ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) IOException(java.io.IOException) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition)

Example 2 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithLocalChannels.

@Test
public void testConsumptionWithLocalChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final PipelinedSubpartition[] partitions = new PipelinedSubpartition[numChannels];
    final Source[] sources = new Source[numChannels];
    final ResultPartitionManager resultPartitionManager = createResultPartitionManager(partitions);
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        partitions[i] = new PipelinedSubpartition(0, resultPartition);
        sources[i] = new PipelinedSubpartitionSource(partitions[i]);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}
Also used : UnregisteredTaskMetricsGroup(org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup) TaskActions(org.apache.flink.runtime.taskmanager.TaskActions) InputChannelTestUtils.createResultPartitionManager(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createResultPartitionManager) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) TaskEventDispatcher(org.apache.flink.runtime.io.network.TaskEventDispatcher) LocalInputChannel(org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel) JobID(org.apache.flink.api.common.JobID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 3 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithRemoteChannels.

@Test
public void testConsumptionWithRemoteChannels() throws Exception {
    final int numChannels = 11;
    final int buffersPerChannel = 1000;
    final ConnectionManager connManager = createDummyConnectionManager();
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        sources[i] = new RemoteChannelSource(channel);
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}
Also used : UnregisteredTaskMetricsGroup(org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup) TaskActions(org.apache.flink.runtime.taskmanager.TaskActions) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) ConnectionID(org.apache.flink.runtime.io.network.ConnectionID) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) InputChannelTestUtils.createDummyConnectionManager(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) JobID(org.apache.flink.api.common.JobID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 4 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironmentTest method testRegisterTaskUsesBoundedBuffers.

/**
	 * Verifies that {@link NetworkEnvironment#registerTask(Task)} sets up (un)bounded buffer pool
	 * instances for various types of input and output channels.
	 */
@Test
public void testRegisterTaskUsesBoundedBuffers() throws Exception {
    final NetworkEnvironment network = new NetworkEnvironment(new NetworkBufferPool(numBuffers, memorySegmentSize, MemoryType.HEAP), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, IOManager.IOMode.SYNC, 0, 0, 2, 8);
    // result partitions
    ResultPartition rp1 = createResultPartition(ResultPartitionType.PIPELINED, 2);
    ResultPartition rp2 = createResultPartition(ResultPartitionType.BLOCKING, 2);
    ResultPartition rp3 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 2);
    ResultPartition rp4 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 8);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    final ResultPartitionWriter[] resultPartitionWriters = new ResultPartitionWriter[] { new ResultPartitionWriter(rp1), new ResultPartitionWriter(rp2), new ResultPartitionWriter(rp3), new ResultPartitionWriter(rp4) };
    // input gates
    final SingleInputGate[] inputGates = new SingleInputGate[] { createSingleInputGateMock(ResultPartitionType.PIPELINED, 2), createSingleInputGateMock(ResultPartitionType.BLOCKING, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 8) };
    // overall task to register
    Task task = mock(Task.class);
    when(task.getProducedPartitions()).thenReturn(resultPartitions);
    when(task.getAllWriters()).thenReturn(resultPartitionWriters);
    when(task.getAllInputGates()).thenReturn(inputGates);
    network.registerTask(task);
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(2 * 2 + 8, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(8 * 2 + 8, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    network.shutdown();
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) Task(org.apache.flink.runtime.taskmanager.Task) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) Test(org.junit.Test)

Example 5 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironmentTest method createSingleInputGateMock.

/**
	 * Helper to create a mock of a {@link SingleInputGate} for use by a {@link Task} inside
	 * {@link NetworkEnvironment#registerTask(Task)}.
	 *
	 * @param partitionType
	 * 		the consumed partition type
	 * @param channels
	 * 		the nummer of input channels
	 *
	 * @return mock with minimal functionality necessary by {@link NetworkEnvironment#registerTask(Task)}
	 */
private static SingleInputGate createSingleInputGateMock(final ResultPartitionType partitionType, final int channels) {
    SingleInputGate ig = mock(SingleInputGate.class);
    when(ig.getConsumedPartitionType()).thenReturn(partitionType);
    when(ig.getNumberOfInputChannels()).thenReturn(channels);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(final InvocationOnMock invocation) throws Throwable {
            BufferPool bp = invocation.getArgumentAt(0, BufferPool.class);
            if (partitionType == ResultPartitionType.PIPELINED_BOUNDED) {
                assertEquals(channels * 2 + 8, bp.getMaxNumberOfMemorySegments());
            } else {
                assertEquals(Integer.MAX_VALUE, bp.getMaxNumberOfMemorySegments());
            }
            return null;
        }
    }).when(ig).setBufferPool(any(BufferPool.class));
    return ig;
}
Also used : BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) InvocationOnMock(org.mockito.invocation.InvocationOnMock) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)

Aggregations

SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)56 Test (org.junit.Test)32 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)30 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)22 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)21 InputChannelTestUtils.createRemoteInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel)15 InputChannelTestUtils.createSingleInputGate (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createSingleInputGate)15 BufferPool (org.apache.flink.runtime.io.network.buffer.BufferPool)14 IntermediateDataSetID (org.apache.flink.runtime.jobgraph.IntermediateDataSetID)11 EmbeddedChannel (org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel)10 InputChannelInfo (org.apache.flink.runtime.checkpoint.channel.InputChannelInfo)8 IOException (java.io.IOException)7 ConnectionManager (org.apache.flink.runtime.io.network.ConnectionManager)7 PartitionRequestClient (org.apache.flink.runtime.io.network.PartitionRequestClient)7 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)7 BufferResponse (org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse)7 InputChannelTestUtils.createDummyConnectionManager (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager)7 InputChannelBuilder (org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder)7 TestInputChannel (org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel)6 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)5