Search in sources :

Example 1 with ResultPartition

use of org.apache.flink.runtime.io.network.partition.ResultPartition in project flink by apache.

the class NetworkEnvironment method unregisterTask.

public void unregisterTask(Task task) {
    LOG.debug("Unregister task {} from network environment (state: {}).", task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());
    final ExecutionAttemptID executionId = task.getExecutionId();
    synchronized (lock) {
        if (isShutdown) {
            // no need to do anything when we are not operational
            return;
        }
        if (task.isCanceledOrFailed()) {
            resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
        }
        ResultPartitionWriter[] writers = task.getAllWriters();
        if (writers != null) {
            for (ResultPartitionWriter writer : writers) {
                taskEventDispatcher.unregisterWriter(writer);
            }
        }
        ResultPartition[] partitions = task.getProducedPartitions();
        if (partitions != null) {
            for (ResultPartition partition : partitions) {
                partition.destroyBufferPool();
            }
        }
        final SingleInputGate[] inputGates = task.getAllInputGates();
        if (inputGates != null) {
            for (SingleInputGate gate : inputGates) {
                try {
                    if (gate != null) {
                        gate.releaseAllResources();
                    }
                } catch (IOException e) {
                    LOG.error("Error during release of reader resources: " + e.getMessage(), e);
                }
            }
        }
    }
}
Also used : ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) IOException(java.io.IOException) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition)

Example 2 with ResultPartition

use of org.apache.flink.runtime.io.network.partition.ResultPartition in project flink by apache.

the class NetworkEnvironmentTest method testRegisterTaskUsesBoundedBuffers.

/**
	 * Verifies that {@link NetworkEnvironment#registerTask(Task)} sets up (un)bounded buffer pool
	 * instances for various types of input and output channels.
	 */
@Test
public void testRegisterTaskUsesBoundedBuffers() throws Exception {
    final NetworkEnvironment network = new NetworkEnvironment(new NetworkBufferPool(numBuffers, memorySegmentSize, MemoryType.HEAP), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, IOManager.IOMode.SYNC, 0, 0, 2, 8);
    // result partitions
    ResultPartition rp1 = createResultPartition(ResultPartitionType.PIPELINED, 2);
    ResultPartition rp2 = createResultPartition(ResultPartitionType.BLOCKING, 2);
    ResultPartition rp3 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 2);
    ResultPartition rp4 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 8);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    final ResultPartitionWriter[] resultPartitionWriters = new ResultPartitionWriter[] { new ResultPartitionWriter(rp1), new ResultPartitionWriter(rp2), new ResultPartitionWriter(rp3), new ResultPartitionWriter(rp4) };
    // input gates
    final SingleInputGate[] inputGates = new SingleInputGate[] { createSingleInputGateMock(ResultPartitionType.PIPELINED, 2), createSingleInputGateMock(ResultPartitionType.BLOCKING, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 8) };
    // overall task to register
    Task task = mock(Task.class);
    when(task.getProducedPartitions()).thenReturn(resultPartitions);
    when(task.getAllWriters()).thenReturn(resultPartitionWriters);
    when(task.getAllInputGates()).thenReturn(inputGates);
    network.registerTask(task);
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(2 * 2 + 8, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(8 * 2 + 8, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    network.shutdown();
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) Task(org.apache.flink.runtime.taskmanager.Task) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) Test(org.junit.Test)

Example 3 with ResultPartition

use of org.apache.flink.runtime.io.network.partition.ResultPartition in project flink by apache.

the class SequentialChannelStateReaderImplTest method collectBuffers.

private Map<ResultSubpartitionInfo, List<Buffer>> collectBuffers(BufferWritingResultPartition[] resultPartitions) throws IOException {
    Map<ResultSubpartitionInfo, List<Buffer>> actual = new HashMap<>();
    for (BufferWritingResultPartition resultPartition : resultPartitions) {
        for (int i = 0; i < resultPartition.getNumberOfSubpartitions(); i++) {
            ResultSubpartitionInfo info = resultPartition.getAllPartitions()[i].getSubpartitionInfo();
            ResultSubpartitionView view = resultPartition.createSubpartitionView(info.getSubPartitionIdx(), new NoOpBufferAvailablityListener());
            for (BufferAndBacklog buffer = view.getNextBuffer(); buffer != null; buffer = view.getNextBuffer()) {
                if (buffer.buffer().isBuffer()) {
                    actual.computeIfAbsent(info, unused -> new ArrayList<>()).add(buffer.buffer());
                }
            }
        }
    }
    return actual;
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog)

Example 4 with ResultPartition

use of org.apache.flink.runtime.io.network.partition.ResultPartition in project flink by apache.

the class SequentialChannelStateReaderImplTest method withResultPartitions.

private void withResultPartitions(ThrowingConsumer<BufferWritingResultPartition[], Exception> action) throws Exception {
    int segmentsToAllocate = parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    BufferWritingResultPartition[] resultPartitions = range(0, parLevel).mapToObj(i -> new ResultPartitionBuilder().setResultPartitionIndex(i).setNumberOfSubpartitions(parLevel).setNetworkBufferPool(networkBufferPool).build()).toArray(BufferWritingResultPartition[]::new);
    try {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.setup();
        }
        action.accept(resultPartitions);
    } finally {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.close();
        }
        try {
            assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
        } finally {
            networkBufferPool.destroyAllBufferPools();
            networkBufferPool.destroy();
        }
    }
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition)

Example 5 with ResultPartition

use of org.apache.flink.runtime.io.network.partition.ResultPartition in project flink by apache.

the class OutputBufferPoolUsageGauge method getValue.

@Override
public Float getValue() {
    int usedBuffers = 0;
    int bufferPoolSize = 0;
    for (ResultPartition resultPartition : resultPartitions) {
        BufferPool bufferPool = resultPartition.getBufferPool();
        if (bufferPool != null) {
            usedBuffers += bufferPool.bestEffortGetNumOfUsedBuffers();
            bufferPoolSize += bufferPool.getNumBuffers();
        }
    }
    if (bufferPoolSize != 0) {
        return ((float) usedBuffers) / bufferPoolSize;
    } else {
        return 0.0f;
    }
}
Also used : BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition)

Aggregations

ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)40 Test (org.junit.Test)23 ResultPartitionWriter (org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter)11 IOException (java.io.IOException)9 NoOpBufferAvailablityListener (org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener)9 ResultSubpartitionView (org.apache.flink.runtime.io.network.partition.ResultSubpartitionView)8 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)8 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)7 CompletingCheckpointResponder (org.apache.flink.streaming.util.CompletingCheckpointResponder)7 Assert.assertEquals (org.junit.Assert.assertEquals)7 Assert.assertTrue (org.junit.Assert.assertTrue)7 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)6 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)6 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)6 ResultPartitionManager (org.apache.flink.runtime.io.network.partition.ResultPartitionManager)6 Collections (java.util.Collections)5 EndOfData (org.apache.flink.runtime.io.network.api.EndOfData)5 ResultPartitionType (org.apache.flink.runtime.io.network.partition.ResultPartitionType)5 ArrayList (java.util.ArrayList)4 CompletableFuture (java.util.concurrent.CompletableFuture)4