Examples with ResultPartitionWriter org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter used on opensource projects

Example 1 with ResultPartitionWriter

use of org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter in project flink by apache.

the class NetworkEnvironment method unregisterTask.

public void unregisterTask(Task task) {
    LOG.debug("Unregister task {} from network environment (state: {}).", task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());
    final ExecutionAttemptID executionId = task.getExecutionId();
    synchronized (lock) {
        if (isShutdown) {
            // no need to do anything when we are not operational
            return;
        }
        if (task.isCanceledOrFailed()) {
            resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
        }
        ResultPartitionWriter[] writers = task.getAllWriters();
        if (writers != null) {
            for (ResultPartitionWriter writer : writers) {
                taskEventDispatcher.unregisterWriter(writer);
            }
        }
        ResultPartition[] partitions = task.getProducedPartitions();
        if (partitions != null) {
            for (ResultPartition partition : partitions) {
                partition.destroyBufferPool();
            }
        }
        final SingleInputGate[] inputGates = task.getAllInputGates();
        if (inputGates != null) {
            for (SingleInputGate gate : inputGates) {
                try {
                    if (gate != null) {
                        gate.releaseAllResources();
                    }
                } catch (IOException e) {
                    LOG.error("Error during release of reader resources: " + e.getMessage(), e);
                }
            }
        }
    }
}

Example 2 with ResultPartitionWriter

use of org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter in project flink by apache.

the class MockEnvironment method addOutput.

public void addOutput(final List<Record> outputList) {
    try {
        // The record-oriented writers wrap the buffer writer. We mock it
        // to collect the returned buffers and deserialize the content to
        // the output list
        BufferProvider mockBufferProvider = mock(BufferProvider.class);
        when(mockBufferProvider.requestBufferBlocking()).thenAnswer(new Answer<Buffer>() {

            @Override
            public Buffer answer(InvocationOnMock invocationOnMock) throws Throwable {
                return new Buffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), mock(BufferRecycler.class));
            }
        });
        ResultPartitionWriter mockWriter = mock(ResultPartitionWriter.class);
        when(mockWriter.getNumberOfOutputChannels()).thenReturn(1);
        when(mockWriter.getBufferProvider()).thenReturn(mockBufferProvider);
        final Record record = new Record();
        final RecordDeserializer<Record> deserializer = new AdaptiveSpanningRecordDeserializer<Record>();
        // Add records from the buffer to the output list
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                deserializer.setNextBuffer(buffer);
                while (deserializer.hasUnfinishedData()) {
                    RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(record);
                    if (result.isFullRecord()) {
                        outputList.add(record.createCopy());
                    }
                    if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER || result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
                        break;
                    }
                }
                return null;
            }
        }).when(mockWriter).writeBuffer(any(Buffer.class), anyInt());
        outputs.add(mockWriter);
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
}

Example 3 with ResultPartitionWriter

use of org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter in project flink by apache.

the class NetworkEnvironmentTest method testRegisterTaskUsesBoundedBuffers.

/**
	 * Verifies that {@link NetworkEnvironment#registerTask(Task)} sets up (un)bounded buffer pool
	 * instances for various types of input and output channels.
	 */
@Test
public void testRegisterTaskUsesBoundedBuffers() throws Exception {
    final NetworkEnvironment network = new NetworkEnvironment(new NetworkBufferPool(numBuffers, memorySegmentSize, MemoryType.HEAP), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, IOManager.IOMode.SYNC, 0, 0, 2, 8);
    // result partitions
    ResultPartition rp1 = createResultPartition(ResultPartitionType.PIPELINED, 2);
    ResultPartition rp2 = createResultPartition(ResultPartitionType.BLOCKING, 2);
    ResultPartition rp3 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 2);
    ResultPartition rp4 = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED, 8);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    final ResultPartitionWriter[] resultPartitionWriters = new ResultPartitionWriter[] { new ResultPartitionWriter(rp1), new ResultPartitionWriter(rp2), new ResultPartitionWriter(rp3), new ResultPartitionWriter(rp4) };
    // input gates
    final SingleInputGate[] inputGates = new SingleInputGate[] { createSingleInputGateMock(ResultPartitionType.PIPELINED, 2), createSingleInputGateMock(ResultPartitionType.BLOCKING, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 2), createSingleInputGateMock(ResultPartitionType.PIPELINED_BOUNDED, 8) };
    // overall task to register
    Task task = mock(Task.class);
    when(task.getProducedPartitions()).thenReturn(resultPartitions);
    when(task.getAllWriters()).thenReturn(resultPartitionWriters);
    when(task.getAllInputGates()).thenReturn(inputGates);
    network.registerTask(task);
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(2 * 2 + 8, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(8 * 2 + 8, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    network.shutdown();
}

Example 4 with ResultPartitionWriter

use of org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter in project flink by apache.

the class StreamRecordWriterTest method testPropagateAsyncFlushError.

/**
	 * Verifies that exceptions during flush from the output flush thread are
	 * recognized in the writer.
	 */
@Test
public void testPropagateAsyncFlushError() {
    FailingWriter<LongValue> testWriter = null;
    try {
        ResultPartitionWriter mockResultPartitionWriter = getMockWriter(5);
        // test writer that flushes every 5ms and fails after 3 flushes
        testWriter = new FailingWriter<LongValue>(mockResultPartitionWriter, new RoundRobinChannelSelector<LongValue>(), 5, 3);
        try {
            // in max 20 seconds (conservative)
            long deadline = System.currentTimeMillis() + 20000;
            long l = 0L;
            while (System.currentTimeMillis() < deadline) {
                testWriter.emit(new LongValue(l++));
            }
            fail("This should have failed with an exception");
        } catch (IOException e) {
            assertNotNull(e.getCause());
            assertTrue(e.getCause().getMessage().contains("Test Exception"));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (testWriter != null) {
            testWriter.close();
        }
    }
}

Example 5 with ResultPartitionWriter

use of org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter in project flink by apache.

the class Task method releaseResources.

/**
 * Releases resources before task exits. We should also fail the partition to release if the
 * task has failed, is canceled, or is being canceled at the moment.
 */
private void releaseResources() {
    LOG.debug("Release task {} network resources (state: {}).", taskNameWithSubtask, getExecutionState());
    for (ResultPartitionWriter partitionWriter : consumableNotifyingPartitionWriters) {
        taskEventDispatcher.unregisterPartition(partitionWriter.getPartitionId());
    }
    // close network resources
    if (isCanceledOrFailed()) {
        failAllResultPartitions();
    }
    closeAllResultPartitions();
    closeAllInputGates();
    try {
        taskStateManager.close();
    } catch (Exception e) {
        LOG.error("Failed to close task state manager for task {}.", taskNameWithSubtask, e);
    }
}