Search in sources :

Example 1 with BufferWritingResultPartition

use of org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition in project flink by apache.

the class SequentialChannelStateReaderImplTest method collectBuffers.

private Map<ResultSubpartitionInfo, List<Buffer>> collectBuffers(BufferWritingResultPartition[] resultPartitions) throws IOException {
    Map<ResultSubpartitionInfo, List<Buffer>> actual = new HashMap<>();
    for (BufferWritingResultPartition resultPartition : resultPartitions) {
        for (int i = 0; i < resultPartition.getNumberOfSubpartitions(); i++) {
            ResultSubpartitionInfo info = resultPartition.getAllPartitions()[i].getSubpartitionInfo();
            ResultSubpartitionView view = resultPartition.createSubpartitionView(info.getSubPartitionIdx(), new NoOpBufferAvailablityListener());
            for (BufferAndBacklog buffer = view.getNextBuffer(); buffer != null; buffer = view.getNextBuffer()) {
                if (buffer.buffer().isBuffer()) {
                    actual.computeIfAbsent(info, unused -> new ArrayList<>()).add(buffer.buffer());
                }
            }
        }
    }
    return actual;
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog)

Example 2 with BufferWritingResultPartition

use of org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition in project flink by apache.

the class SequentialChannelStateReaderImplTest method withResultPartitions.

private void withResultPartitions(ThrowingConsumer<BufferWritingResultPartition[], Exception> action) throws Exception {
    int segmentsToAllocate = parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    BufferWritingResultPartition[] resultPartitions = range(0, parLevel).mapToObj(i -> new ResultPartitionBuilder().setResultPartitionIndex(i).setNumberOfSubpartitions(parLevel).setNetworkBufferPool(networkBufferPool).build()).toArray(BufferWritingResultPartition[]::new);
    try {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.setup();
        }
        action.accept(resultPartitions);
    } finally {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.close();
        }
        try {
            assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
        } finally {
            networkBufferPool.destroyAllBufferPools();
            networkBufferPool.destroy();
        }
    }
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition)

Example 3 with BufferWritingResultPartition

use of org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition in project flink by apache.

the class SingleInputGateTest method testQueuedBuffers.

@Test
public void testQueuedBuffers() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final BufferWritingResultPartition resultPartition = (BufferWritingResultPartition) new ResultPartitionBuilder().setResultPartitionManager(network.getResultPartitionManager()).setupBufferPoolFactoryFromNettyShuffleEnvironment(network).build();
    final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);
    final ResultPartitionID localResultPartitionId = resultPartition.getPartitionId();
    final InputChannel[] inputChannels = new InputChannel[2];
    final RemoteInputChannel remoteInputChannel = InputChannelBuilder.newBuilder().setChannelIndex(1).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
    inputChannels[0] = remoteInputChannel;
    inputChannels[1] = InputChannelBuilder.newBuilder().setChannelIndex(0).setPartitionId(localResultPartitionId).setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildLocalChannel(inputGate);
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        closer.register(resultPartition::release);
        resultPartition.setup();
        setupInputGate(inputGate, inputChannels);
        remoteInputChannel.onBuffer(createBuffer(1), 0, 0);
        assertEquals(1, inputGate.getNumberOfQueuedBuffers());
        resultPartition.emitRecord(ByteBuffer.allocate(1), 0);
        assertEquals(2, inputGate.getNumberOfQueuedBuffers());
    }
}
Also used : Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 4 with BufferWritingResultPartition

use of org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition in project flink by apache.

the class ChannelPersistenceITCase method upstreamBlocksAfterRecoveringState.

private void upstreamBlocksAfterRecoveringState(ResultPartitionType type) throws Exception {
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    byte[] dataAfterRecovery = randomBytes(1024);
    try {
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, type, 0, 1);
        new SequentialChannelStateReaderImpl(new TaskStateSnapshot()).readOutputData(new BufferWritingResultPartition[] { resultPartition }, true);
        resultPartition.emitRecord(ByteBuffer.wrap(dataAfterRecovery), 0);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        if (type != ResultPartitionType.PIPELINED_APPROXIMATE) {
            assertEquals(RECOVERY_COMPLETION, view.getNextBuffer().buffer().getDataType());
            assertNull(view.getNextBuffer());
            view.resumeConsumption();
        }
        assertArrayEquals(dataAfterRecovery, collectBytes(view.getNextBuffer().buffer()));
    } finally {
        networkBufferPool.destroy();
    }
}
Also used : TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) SequentialChannelStateReaderImpl(org.apache.flink.runtime.checkpoint.channel.SequentialChannelStateReaderImpl) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)

Example 5 with BufferWritingResultPartition

use of org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition in project flink by apache.

the class LocalInputChannelTest method testConcurrentConsumeMultiplePartitions.

/**
 * Tests the consumption of multiple subpartitions via local input channels.
 *
 * <p>Multiple producer tasks produce pipelined partitions, which are consumed by multiple tasks
 * via local input channels.
 */
@Test
public void testConcurrentConsumeMultiplePartitions() throws Exception {
    // Config
    final int parallelism = 32;
    final int producerBufferPoolSize = parallelism + 1;
    final int numberOfBuffersPerChannel = 1024;
    // Setup
    // One thread per produced partition and one per consumer
    final ExecutorService executor = Executors.newFixedThreadPool(2 * parallelism);
    final NetworkBufferPool networkBuffers = new NetworkBufferPool((parallelism * producerBufferPoolSize) + (parallelism * parallelism), TestBufferFactory.BUFFER_SIZE);
    final ResultPartitionManager partitionManager = new ResultPartitionManager();
    final ResultPartitionID[] partitionIds = new ResultPartitionID[parallelism];
    final TestPartitionProducer[] partitionProducers = new TestPartitionProducer[parallelism];
    // Create all partitions
    for (int i = 0; i < parallelism; i++) {
        partitionIds[i] = new ResultPartitionID();
        final ResultPartition partition = new ResultPartitionBuilder().setResultPartitionId(partitionIds[i]).setNumberOfSubpartitions(parallelism).setNumTargetKeyGroups(parallelism).setResultPartitionManager(partitionManager).setBufferPoolFactory(() -> networkBuffers.createBufferPool(producerBufferPoolSize, producerBufferPoolSize, parallelism, Integer.MAX_VALUE)).build();
        // Create a buffer pool for this partition
        partition.setup();
        // Create the producer
        partitionProducers[i] = new TestPartitionProducer((BufferWritingResultPartition) partition, false, new TestPartitionProducerBufferSource(parallelism, TestBufferFactory.BUFFER_SIZE, numberOfBuffersPerChannel));
    }
    // Test
    try {
        // Submit producer tasks
        List<CompletableFuture<?>> results = Lists.newArrayListWithCapacity(parallelism + 1);
        for (int i = 0; i < parallelism; i++) {
            results.add(CompletableFuture.supplyAsync(CheckedSupplier.unchecked(partitionProducers[i]::call), executor));
        }
        // Submit consumer
        for (int i = 0; i < parallelism; i++) {
            final TestLocalInputChannelConsumer consumer = new TestLocalInputChannelConsumer(i, parallelism, numberOfBuffersPerChannel, networkBuffers.createBufferPool(parallelism, parallelism), partitionManager, new TaskEventDispatcher(), partitionIds);
            results.add(CompletableFuture.supplyAsync(CheckedSupplier.unchecked(consumer::call), executor));
        }
        FutureUtils.waitForAll(results).get();
    } finally {
        networkBuffers.destroyAllBufferPools();
        networkBuffers.destroy();
        executor.shutdown();
    }
}
Also used : TestPartitionProducer(org.apache.flink.runtime.io.network.util.TestPartitionProducer) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) TestingResultPartitionManager(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateTest.TestingResultPartitionManager) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) CompletableFuture(java.util.concurrent.CompletableFuture) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ExecutorService(java.util.concurrent.ExecutorService) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) TaskEventDispatcher(org.apache.flink.runtime.io.network.TaskEventDispatcher) Test(org.junit.Test)

Aggregations

BufferWritingResultPartition (org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition)7 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)5 ResultPartitionBuilder (org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder)5 Test (org.junit.Test)5 NoOpBufferAvailablityListener (org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener)4 ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)4 ResultSubpartitionView (org.apache.flink.runtime.io.network.partition.ResultSubpartitionView)4 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)3 InputGate (org.apache.flink.runtime.io.network.partition.consumer.InputGate)3 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)3 Closer (org.apache.flink.shaded.guava30.com.google.common.io.Closer)3 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 DataOutputStream (java.io.DataOutputStream)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Optional (java.util.Optional)2