Examples with RandomAccessOutputView org.apache.flink.runtime.io.disk.RandomAccessOutputView used on opensource projects

Example 1 with RandomAccessOutputView

use of org.apache.flink.runtime.io.disk.RandomAccessOutputView in project flink by apache.

the class BinaryRowDataTest method testHashAndCopy.

@Test
public void testHashAndCopy() throws IOException {
    MemorySegment[] segments = new MemorySegment[3];
    for (int i = 0; i < 3; i++) {
        segments[i] = MemorySegmentFactory.wrap(new byte[64]);
    }
    RandomAccessOutputView out = new RandomAccessOutputView(segments, 64);
    BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(2);
    BinaryRowData row = new BinaryRowData(2);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    writer.writeString(0, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahah"));
    writer.writeString(1, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahaa"));
    writer.complete();
    serializer.serializeToPages(row, out);
    ArrayList<MemorySegment> segmentList = new ArrayList<>(Arrays.asList(segments));
    RandomAccessInputView input = new RandomAccessInputView(segmentList, 64, 64);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, input);
    assertEquals(row, mapRow);
    assertEquals(row.getString(0), mapRow.getString(0));
    assertEquals(row.getString(1), mapRow.getString(1));
    assertNotEquals(row.getString(0), mapRow.getString(1));
    // test if the hash code before and after serialization are the same
    assertEquals(row.hashCode(), mapRow.hashCode());
    assertEquals(row.getString(0).hashCode(), mapRow.getString(0).hashCode());
    assertEquals(row.getString(1).hashCode(), mapRow.getString(1).hashCode());
    // test if the copy method produce a row with the same contents
    assertEquals(row.copy(), mapRow.copy());
    assertEquals(((BinaryStringData) row.getString(0)).copy(), ((BinaryStringData) mapRow.getString(0)).copy());
    assertEquals(((BinaryStringData) row.getString(1)).copy(), ((BinaryStringData) mapRow.getString(1)).copy());
}

Example 2 with RandomAccessOutputView

use of org.apache.flink.runtime.io.disk.RandomAccessOutputView in project flink by apache.

the class BinaryRowDataTest method testGenericObject.

@Test
public void testGenericObject() throws Exception {
    GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
    TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());
    RawValueDataSerializer<MyObj> binarySerializer = new RawValueDataSerializer<>(genericSerializer);
    BinaryRowData row = new BinaryRowData(4);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    writer.writeInt(0, 0);
    RawValueData<MyObj> myObj1 = RawValueData.fromObject(new MyObj(0, 1));
    writer.writeRawValue(1, myObj1, binarySerializer);
    RawValueData<MyObj> myObj2 = RawValueData.fromObject(new MyObj(123, 5.0));
    writer.writeRawValue(2, myObj2, binarySerializer);
    RawValueData<MyObj> myObj3 = RawValueData.fromObject(new MyObj(1, 1));
    writer.writeRawValue(3, myObj3, binarySerializer);
    writer.complete();
    assertTestGenericObjectRow(row, genericSerializer);
    // getBytes from var-length memorySegments.
    BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(4);
    MemorySegment[] memorySegments = new MemorySegment[3];
    ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
    for (int i = 0; i < 3; i++) {
        memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
        memorySegmentList.add(memorySegments[i]);
    }
    RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
    serializer.serializeToPages(row, out);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, new RandomAccessInputView(memorySegmentList, 64));
    assertTestGenericObjectRow(mapRow, genericSerializer);
}

Example 3 with RandomAccessOutputView

use of org.apache.flink.runtime.io.disk.RandomAccessOutputView in project flink by apache.

the class BinaryRowDataTest method testSerStringToKryo.

@Test
public void testSerStringToKryo() throws IOException {
    KryoSerializer<BinaryStringData> serializer = new KryoSerializer<>(BinaryStringData.class, new ExecutionConfig());
    BinaryStringData string = BinaryStringData.fromString("hahahahaha");
    RandomAccessOutputView out = new RandomAccessOutputView(new MemorySegment[] { MemorySegmentFactory.wrap(new byte[1024]) }, 64);
    serializer.serialize(string, out);
    RandomAccessInputView input = new RandomAccessInputView(new ArrayList<>(Collections.singletonList(out.getCurrentSegment())), 64, 64);
    StringData newStr = serializer.deserialize(input);
    assertEquals(string, newStr);
}

Example 4 with RandomAccessOutputView

use of org.apache.flink.runtime.io.disk.RandomAccessOutputView in project flink by apache.

the class BinaryRowDataTest method testSerialize.

private void testSerialize(BinaryRowData row, MemorySegment[] memorySegments, Map<MemorySegment, Integer> msIndex, BinaryRowDataSerializer serializer, int position, int rowSize) throws IOException {
    RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
    out.skipBytesToWrite(position);
    row.setTotalSize(rowSize);
    // this `row` contains random bytes, and now we're going to serialize `rowSize` bytes
    // (not including the row header) of the contents
    serializer.serializeToPages(row, out);
    // let's see how many segments we have written
    int segNumber = msIndex.get(out.getCurrentSegment()) + 1;
    int lastSegSize = out.getCurrentPositionInSegment();
    // now deserialize from the written segments
    ArrayList<MemorySegment> segments = new ArrayList<>(Arrays.asList(memorySegments).subList(0, segNumber));
    RandomAccessInputView input = new RandomAccessInputView(segments, 64, lastSegSize);
    input.skipBytesToRead(position);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, input);
    assertEquals(row, mapRow);
}

Example 5 with RandomAccessOutputView

use of org.apache.flink.runtime.io.disk.RandomAccessOutputView in project flink by apache.

the class BinaryRowDataTest method testPagesSer.

@Test
public void testPagesSer() throws IOException {
    MemorySegment[] memorySegments = new MemorySegment[5];
    ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
        memorySegmentList.add(memorySegments[i]);
    }
    {
        // multi memorySegments
        String str = "啦啦啦啦啦我是快乐的粉刷匠，啦啦啦啦啦我是快乐的粉刷匠，" + "啦啦啦啦啦我是快乐的粉刷匠。";
        BinaryRowData row = new BinaryRowData(1);
        BinaryRowWriter writer = new BinaryRowWriter(row);
        writer.writeString(0, fromString(str));
        writer.complete();
        RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
        BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(1);
        serializer.serializeToPages(row, out);
        BinaryRowData mapRow = serializer.createInstance();
        mapRow = serializer.mapFromPages(mapRow, new RandomAccessInputView(memorySegmentList, 64));
        writer.reset();
        writer.writeString(0, mapRow.getString(0));
        writer.complete();
        assertEquals(str, row.getString(0).toString());
        BinaryRowData deserRow = serializer.deserializeFromPages(new RandomAccessInputView(memorySegmentList, 64));
        writer.reset();
        writer.writeString(0, deserRow.getString(0));
        writer.complete();
        assertEquals(str, row.getString(0).toString());
    }
    {
        // multi memorySegments
        String str1 = "啦啦啦啦啦我是快乐的粉刷匠，啦啦啦啦啦我是快乐的粉刷匠，" + "啦啦啦啦啦我是快乐的粉刷匠。";
        String str2 = "啦啦啦啦啦我是快乐的粉刷匠。";
        BinaryRowData row = new BinaryRowData(2);
        BinaryRowWriter writer = new BinaryRowWriter(row);
        writer.writeString(0, fromString(str1));
        writer.writeString(1, fromString(str2));
        writer.complete();
        RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
        out.skipBytesToWrite(40);
        BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(2);
        serializer.serializeToPages(row, out);
        RandomAccessInputView in = new RandomAccessInputView(memorySegmentList, 64);
        in.skipBytesToRead(40);
        BinaryRowData mapRow = serializer.createInstance();
        mapRow = serializer.mapFromPages(mapRow, in);
        writer.reset();
        writer.writeString(0, mapRow.getString(0));
        writer.writeString(1, mapRow.getString(1));
        writer.complete();
        assertEquals(str1, row.getString(0).toString());
        assertEquals(str2, row.getString(1).toString());
        in = new RandomAccessInputView(memorySegmentList, 64);
        in.skipBytesToRead(40);
        BinaryRowData deserRow = serializer.deserializeFromPages(in);
        writer.reset();
        writer.writeString(0, deserRow.getString(0));
        writer.writeString(1, deserRow.getString(1));
        writer.complete();
        assertEquals(str1, row.getString(0).toString());
        assertEquals(str2, row.getString(1).toString());
    }
}