Examples with RandomAccessInputView org.apache.flink.runtime.io.disk.RandomAccessInputView used on opensource projects

Example 6 with RandomAccessInputView

use of org.apache.flink.runtime.io.disk.RandomAccessInputView in project flink by apache.

the class BinaryRowDataTest method testSerialize.

private void testSerialize(BinaryRowData row, MemorySegment[] memorySegments, Map<MemorySegment, Integer> msIndex, BinaryRowDataSerializer serializer, int position, int rowSize) throws IOException {
    RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
    out.skipBytesToWrite(position);
    row.setTotalSize(rowSize);
    // this `row` contains random bytes, and now we're going to serialize `rowSize` bytes
    // (not including the row header) of the contents
    serializer.serializeToPages(row, out);
    // let's see how many segments we have written
    int segNumber = msIndex.get(out.getCurrentSegment()) + 1;
    int lastSegSize = out.getCurrentPositionInSegment();
    // now deserialize from the written segments
    ArrayList<MemorySegment> segments = new ArrayList<>(Arrays.asList(memorySegments).subList(0, segNumber));
    RandomAccessInputView input = new RandomAccessInputView(segments, 64, lastSegSize);
    input.skipBytesToRead(position);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, input);
    assertEquals(row, mapRow);
}

Example 7 with RandomAccessInputView

use of org.apache.flink.runtime.io.disk.RandomAccessInputView in project flink by apache.

the class BinaryRowDataTest method testPagesSer.

@Test
public void testPagesSer() throws IOException {
    MemorySegment[] memorySegments = new MemorySegment[5];
    ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
        memorySegmentList.add(memorySegments[i]);
    }
    {
        // multi memorySegments
        String str = "啦啦啦啦啦我是快乐的粉刷匠，啦啦啦啦啦我是快乐的粉刷匠，" + "啦啦啦啦啦我是快乐的粉刷匠。";
        BinaryRowData row = new BinaryRowData(1);
        BinaryRowWriter writer = new BinaryRowWriter(row);
        writer.writeString(0, fromString(str));
        writer.complete();
        RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
        BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(1);
        serializer.serializeToPages(row, out);
        BinaryRowData mapRow = serializer.createInstance();
        mapRow = serializer.mapFromPages(mapRow, new RandomAccessInputView(memorySegmentList, 64));
        writer.reset();
        writer.writeString(0, mapRow.getString(0));
        writer.complete();
        assertEquals(str, row.getString(0).toString());
        BinaryRowData deserRow = serializer.deserializeFromPages(new RandomAccessInputView(memorySegmentList, 64));
        writer.reset();
        writer.writeString(0, deserRow.getString(0));
        writer.complete();
        assertEquals(str, row.getString(0).toString());
    }
    {
        // multi memorySegments
        String str1 = "啦啦啦啦啦我是快乐的粉刷匠，啦啦啦啦啦我是快乐的粉刷匠，" + "啦啦啦啦啦我是快乐的粉刷匠。";
        String str2 = "啦啦啦啦啦我是快乐的粉刷匠。";
        BinaryRowData row = new BinaryRowData(2);
        BinaryRowWriter writer = new BinaryRowWriter(row);
        writer.writeString(0, fromString(str1));
        writer.writeString(1, fromString(str2));
        writer.complete();
        RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
        out.skipBytesToWrite(40);
        BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(2);
        serializer.serializeToPages(row, out);
        RandomAccessInputView in = new RandomAccessInputView(memorySegmentList, 64);
        in.skipBytesToRead(40);
        BinaryRowData mapRow = serializer.createInstance();
        mapRow = serializer.mapFromPages(mapRow, in);
        writer.reset();
        writer.writeString(0, mapRow.getString(0));
        writer.writeString(1, mapRow.getString(1));
        writer.complete();
        assertEquals(str1, row.getString(0).toString());
        assertEquals(str2, row.getString(1).toString());
        in = new RandomAccessInputView(memorySegmentList, 64);
        in.skipBytesToRead(40);
        BinaryRowData deserRow = serializer.deserializeFromPages(in);
        writer.reset();
        writer.writeString(0, deserRow.getString(0));
        writer.writeString(1, deserRow.getString(1));
        writer.complete();
        assertEquals(str1, row.getString(0).toString());
        assertEquals(str2, row.getString(1).toString());
    }
}