Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 31 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BackPressureStatsTrackerITCase method testBackPressuredProducer.

/**
	 * Tests a simple fake-back pressured task. Back pressure is assumed when
	 * sampled stack traces are in blocking buffer requests.
	 */
@Test
public void testBackPressuredProducer() throws Exception {
    new JavaTestKit(testActorSystem) {

        {
            final FiniteDuration deadline = new FiniteDuration(60, TimeUnit.SECONDS);
            // The JobGraph
            final JobGraph jobGraph = new JobGraph();
            final int parallelism = 4;
            final JobVertex task = new JobVertex("Task");
            task.setInvokableClass(BackPressuredTask.class);
            task.setParallelism(parallelism);
            jobGraph.addVertex(task);
            ActorGateway jobManger = null;
            ActorGateway taskManager = null;
            //
            // 1) Consume all buffers at first (no buffers for the test task)
            //
            testBufferPool = networkBufferPool.createBufferPool(1, Integer.MAX_VALUE);
            final List<Buffer> buffers = new ArrayList<>();
            while (true) {
                Buffer buffer = testBufferPool.requestBuffer();
                if (buffer != null) {
                    buffers.add(buffer);
                } else {
                    break;
                }
            }
            try {
                jobManger = TestingUtils.createJobManager(testActorSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), new Configuration());
                final Configuration config = new Configuration();
                config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
                taskManager = TestingUtils.createTaskManager(testActorSystem, jobManger, config, true, true);
                final ActorGateway jm = jobManger;
                new Within(deadline) {

                    @Override
                    protected void run() {
                        try {
                            ActorGateway testActor = new AkkaActorGateway(getTestActor(), null);
                            // Submit the job and wait until it is running
                            JobClient.submitJobDetached(jm, config, jobGraph, deadline, ClassLoader.getSystemClassLoader());
                            jm.tell(new WaitForAllVerticesToBeRunning(jobGraph.getJobID()), testActor);
                            expectMsgEquals(new AllVerticesRunning(jobGraph.getJobID()));
                            // Get the ExecutionGraph
                            jm.tell(new RequestExecutionGraph(jobGraph.getJobID()), testActor);
                            ExecutionGraphFound executionGraphResponse = expectMsgClass(ExecutionGraphFound.class);
                            ExecutionGraph executionGraph = (ExecutionGraph) executionGraphResponse.executionGraph();
                            ExecutionJobVertex vertex = executionGraph.getJobVertex(task.getID());
                            StackTraceSampleCoordinator coordinator = new StackTraceSampleCoordinator(testActorSystem.dispatcher(), 60000);
                            // Verify back pressure (clean up interval can be ignored)
                            BackPressureStatsTracker statsTracker = new BackPressureStatsTracker(coordinator, 100 * 1000, 20, Time.milliseconds(10L));
                            int numAttempts = 10;
                            int nextSampleId = 0;
                            // the buffer.
                            for (int attempt = 0; attempt < numAttempts; attempt++) {
                                try {
                                    OperatorBackPressureStats stats = triggerStatsSample(statsTracker, vertex);
                                    assertEquals(nextSampleId + attempt, stats.getSampleId());
                                    assertEquals(parallelism, stats.getNumberOfSubTasks());
                                    assertEquals(1.0, stats.getMaxBackPressureRatio(), 0.0);
                                    for (int i = 0; i < parallelism; i++) {
                                        assertEquals(1.0, stats.getBackPressureRatio(i), 0.0);
                                    }
                                    nextSampleId = stats.getSampleId() + 1;
                                    break;
                                } catch (Throwable t) {
                                    if (attempt == numAttempts - 1) {
                                        throw t;
                                    } else {
                                        Thread.sleep(500);
                                    }
                                }
                            }
                            //
                            for (Buffer buf : buffers) {
                                buf.recycle();
                            }
                            // grab them and then immediately release them.
                            while (testBufferPool.getNumberOfAvailableMemorySegments() < 100) {
                                Thread.sleep(100);
                            }
                            // Verify that no task is back pressured any more.
                            for (int attempt = 0; attempt < numAttempts; attempt++) {
                                try {
                                    OperatorBackPressureStats stats = triggerStatsSample(statsTracker, vertex);
                                    assertEquals(nextSampleId + attempt, stats.getSampleId());
                                    assertEquals(parallelism, stats.getNumberOfSubTasks());
                                    // Verify that no task is back pressured
                                    for (int i = 0; i < parallelism; i++) {
                                        assertEquals(0.0, stats.getBackPressureRatio(i), 0.0);
                                    }
                                    break;
                                } catch (Throwable t) {
                                    if (attempt == numAttempts - 1) {
                                        throw t;
                                    } else {
                                        Thread.sleep(500);
                                    }
                                }
                            }
                            // Shut down
                            jm.tell(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobGraph.getJobID()), testActor);
                            // Cancel job
                            jm.tell(new JobManagerMessages.CancelJob(jobGraph.getJobID()));
                            // Response to removal notification
                            expectMsgEquals(true);
                            //
                            // 3) Trigger stats for archived job
                            //
                            statsTracker.invalidateOperatorStatsCache();
                            assertFalse("Unexpected trigger", statsTracker.triggerStackTraceSample(vertex));
                        } catch (Exception e) {
                            e.printStackTrace();
                            fail(e.getMessage());
                        }
                    }
                };
            } finally {
                TestingUtils.stopActor(jobManger);
                TestingUtils.stopActor(taskManager);
                for (Buffer buf : buffers) {
                    buf.recycle();
                }
                testBufferPool.lazyDestroy();
            }
        }
    };
}

Example 32 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class TestInputChannel method readBuffer.

public TestInputChannel readBuffer() throws IOException, InterruptedException {
    final Buffer buffer = mock(Buffer.class);
    when(buffer.isBuffer()).thenReturn(true);
    return read(buffer);
}

Example 33 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class LargeRecordsTest method testHandleMixedLargeRecords.

@Test
public void testHandleMixedLargeRecords() {
    try {
        final int NUM_RECORDS = 99;
        final int SEGMENT_SIZE = 32 * 1024;
        final RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
        final RecordDeserializer<SerializationTestType> deserializer = new AdaptiveSpanningRecordDeserializer<SerializationTestType>();
        final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
        List<SerializationTestType> originalRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        List<SerializationTestType> deserializedRecords = new ArrayList<SerializationTestType>((NUM_RECORDS + 1) / 2);
        LargeObjectType genLarge = new LargeObjectType();
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            if (i % 2 == 0) {
                originalRecords.add(new IntType(42));
                deserializedRecords.add(new IntType());
            } else {
                originalRecords.add(genLarge.getRandom(rnd));
                deserializedRecords.add(new LargeObjectType());
            }
        }
        // -------------------------------------------------------------------------------------------------------------
        serializer.setNextBuffer(buffer);
        int numRecordsDeserialized = 0;
        for (SerializationTestType record : originalRecords) {
            // serialize record
            if (serializer.addRecord(record).isFullBuffer()) {
                // buffer is full => move to deserializer
                deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                // deserialize records, as many complete as there are
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
                // move buffers as long as necessary (for long records)
                while (serializer.setNextBuffer(buffer).isFullBuffer()) {
                    deserializer.setNextMemorySegment(serializer.getCurrentBuffer().getMemorySegment(), SEGMENT_SIZE);
                }
                // deserialize records, as many as there are in the last buffer
                while (numRecordsDeserialized < deserializedRecords.size()) {
                    SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
                    if (deserializer.getNextRecord(next).isFullRecord()) {
                        assertEquals(originalRecords.get(numRecordsDeserialized), next);
                        numRecordsDeserialized++;
                    } else {
                        break;
                    }
                }
            }
        }
        // move the last (incomplete buffer)
        Buffer last = serializer.getCurrentBuffer();
        deserializer.setNextMemorySegment(last.getMemorySegment(), last.getSize());
        serializer.clear();
        // deserialize records, as many as there are in the last buffer
        while (numRecordsDeserialized < deserializedRecords.size()) {
            SerializationTestType next = deserializedRecords.get(numRecordsDeserialized);
            assertTrue(deserializer.getNextRecord(next).isFullRecord());
            assertEquals(originalRecords.get(numRecordsDeserialized), next);
            numRecordsDeserialized++;
        }
        // might be that the last big records has not yet been fully moved, and a small one is missing
        assertFalse(serializer.hasData());
        assertFalse(deserializer.hasUnfinishedData());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 34 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestClientHandler method decodeBufferOrEvent.

private boolean decodeBufferOrEvent(RemoteInputChannel inputChannel, NettyMessage.BufferResponse bufferOrEvent, boolean isStagedBuffer) throws Throwable {
    boolean releaseNettyBuffer = true;
    try {
        if (bufferOrEvent.isBuffer()) {
            // IndexOutOfBoundsException.
            if (bufferOrEvent.getSize() == 0) {
                inputChannel.onEmptyBuffer(bufferOrEvent.sequenceNumber);
                return true;
            }
            BufferProvider bufferProvider = inputChannel.getBufferProvider();
            if (bufferProvider == null) {
                // receiver has been cancelled/failed
                cancelRequestFor(bufferOrEvent.receiverId);
                return isStagedBuffer;
            }
            while (true) {
                Buffer buffer = bufferProvider.requestBuffer();
                if (buffer != null) {
                    buffer.setSize(bufferOrEvent.getSize());
                    bufferOrEvent.getNettyBuffer().readBytes(buffer.getNioBuffer());
                    inputChannel.onBuffer(buffer, bufferOrEvent.sequenceNumber);
                    return true;
                } else if (bufferListener.waitForBuffer(bufferProvider, bufferOrEvent)) {
                    releaseNettyBuffer = false;
                    return false;
                } else if (bufferProvider.isDestroyed()) {
                    return isStagedBuffer;
                }
            }
        } else {
            // ---- Event -------------------------------------------------
            // TODO We can just keep the serialized data in the Netty buffer and release it later at the reader
            byte[] byteArray = new byte[bufferOrEvent.getSize()];
            bufferOrEvent.getNettyBuffer().readBytes(byteArray);
            MemorySegment memSeg = MemorySegmentFactory.wrap(byteArray);
            Buffer buffer = new Buffer(memSeg, FreeingBufferRecycler.INSTANCE, false);
            inputChannel.onBuffer(buffer, bufferOrEvent.sequenceNumber);
            return true;
        }
    } finally {
        if (releaseNettyBuffer) {
            bufferOrEvent.releaseBuffer();
        }
    }
}

Example 35 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class EventSerializer method toBuffer.

// ------------------------------------------------------------------------
// Buffer helpers
// ------------------------------------------------------------------------
public static Buffer toBuffer(AbstractEvent event) throws IOException {
    final ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);
    MemorySegment data = MemorySegmentFactory.wrap(serializedEvent.array());
    final Buffer buffer = new Buffer(data, FreeingBufferRecycler.INSTANCE, false);
    buffer.setSize(serializedEvent.remaining());
    return buffer;
}