Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 71 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class DemultiplexingRecordDeserializerTest method testAmbiguousChannels.

/**
 * Tests that {@link RecordFilter} are used correctly.
 */
@Test
public void testAmbiguousChannels() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(1, 0), rescalingDescriptor(to(41, 42), array(mappings(), mappings(to(2, 3), to(4, 5))), set(42)), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> new RecordFilter(new ModSelector(2), LongSerializer.INSTANCE, 1));
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(41, 2), new SubtaskConnectionDescriptor(41, 3), new SubtaskConnectionDescriptor(42, 2), new SubtaskConnectionDescriptor(42, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            // add one even and one odd number
            Buffer buffer = writeLongs(bufferBuilder, i, i + 1L);
            SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), i / 10 % 2);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
            if (selector.getInputSubtaskIndex() == 41) {
                assertEquals(Arrays.asList((long) i, i + 1L), readLongs(deserializer));
            } else {
                // only odd should occur in output
                assertEquals(Arrays.asList(i / 2 * 2 + 1L), readLongs(deserializer));
            }
        }
        assertTrue(memorySegment.isFreed());
    }
}

Example 72 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class DemultiplexingRecordDeserializerTest method testUpscale.

/**
 * Tests {@link SubtaskConnectionDescriptor} by mixing buffers from 4 different virtual
 * channels.
 */
@Test
public void testUpscale() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(2, 0), rescalingDescriptor(to(0, 1), array(mappings(), mappings(), mappings(to(2, 3), to(4, 5))), emptySet()), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> RecordFilter.all());
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(0, 2), new SubtaskConnectionDescriptor(0, 3), new SubtaskConnectionDescriptor(1, 2), new SubtaskConnectionDescriptor(1, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), random.nextInt(4));
        long start = selector.getInputSubtaskIndex() << 4 | selector.getOutputSubtaskIndex();
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            Buffer buffer = writeLongs(bufferBuilder, start + 1L, start + 2L, start + 3L);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
        }
        assertEquals(Arrays.asList(start + 1L, start + 2L, start + 3L), readLongs(deserializer));
        assertTrue(memorySegment.isFreed());
    }
}

Example 73 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnFirstBarrier.

/**
 * This test tries to make sure that the first time out happens after processing {@link
 * EventAnnouncement} but before/during processing the first {@link CheckpointBarrier}.
 */
@Test
public void testTimeoutAlignmentOnFirstBarrier() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    for (int i = 0; i < numChannels; i++) {
        (getChannel(gate, i)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    }
    assertEquals(0, target.getTriggeredCheckpointCounter());
    for (int i = 0; i < numChannels; i++) {
        assertAnnouncement(gate);
    }
    assertEquals(0, target.getTriggeredCheckpointCounter());
    clock.advanceTime(alignmentTimeout * 4, TimeUnit.MILLISECONDS);
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 74 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutBarrierOnTwoChannels.

private void testTimeoutBarrierOnTwoChannels(ValidatingCheckpointHandler target, CheckpointedInputGate gate, long alignmentTimeout) throws Exception {
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    getChannel(gate, 0).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 0).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 0).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
    assertEquals(0, target.getTriggeredCheckpointCounter());
    assertAnnouncement(gate);
    clock.advanceTime(alignmentTimeout * 2, TimeUnit.MILLISECONDS);
    assertAnnouncement(gate);
    assertBarrier(gate);
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
    assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
    // Followed by overtaken buffers
    assertData(gate);
    assertData(gate);
    assertData(gate);
}

Example 75 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method performFirstCheckpoint.

private void performFirstCheckpoint(int numChannels, ValidatingCheckpointHandler target, CheckpointedInputGate gate, long alignmentTimeout) throws IOException, InterruptedException {
    Buffer checkpointBarrier = withTimeout(1, alignmentTimeout);
    for (int i = 0; i < numChannels; i++) {
        (getChannel(gate, i)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    }
    assertEquals(0, target.getTriggeredCheckpointCounter());
    for (int i = 0; i < numChannels; i++) {
        assertAnnouncement(gate);
    }
    for (int i = 0; i < numChannels; i++) {
        assertBarrier(gate);
    }
}