Examples with BinaryRowDataSerializer org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer used on opensource projects

Example 6 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class SortMergeJoinIteratorTest method fullOuter.

public void fullOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator<BinaryRowData> input1 = data.f0;
    MutableObjectIterator<BinaryRowData> input2 = data.f1;
    try (SortMergeFullOuterJoinIterator iterator = new SortMergeFullOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            BinaryRowData matchKey = iterator.getMatchKey();
            ResettableExternalBuffer buffer1 = iterator.getBuffer1();
            ResettableExternalBuffer buffer2 = iterator.getBuffer2();
            if (matchKey == null && buffer1.size() > 0) {
                // left outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer1.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, null));
                }
            } else if (matchKey == null && buffer2.size() > 0) {
                // right outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer2.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(null, row));
                }
            } else if (matchKey != null) {
                // match join.
                ResettableExternalBuffer.BufferIterator iter1 = buffer1.newIterator();
                while (iter1.advanceNext()) {
                    RowData row1 = iter1.getRow();
                    ResettableExternalBuffer.BufferIterator iter2 = buffer2.newIterator();
                    while (iter2.advanceNext()) {
                        RowData row2 = iter2.getRow();
                        Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                        assertEquals(expected, new Tuple2<>(row1, row2));
                    }
                }
            } else {
                // bug...
                throw new RuntimeException("There is a bug.");
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 7 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class SortOperator method open.

@Override
public void open() throws Exception {
    super.open();
    LOG.info("Opening SortOperator");
    ClassLoader cl = getContainingTask().getUserCodeClassLoader();
    AbstractRowDataSerializer inputSerializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader());
    this.binarySerializer = new BinaryRowDataSerializer(inputSerializer.getArity());
    NormalizedKeyComputer computer = gComputer.newInstance(cl);
    RecordComparator comparator = gComparator.newInstance(cl);
    gComputer = null;
    gComparator = null;
    MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
    this.sorter = new BinaryExternalSorter(this.getContainingTask(), memManager, computeMemorySize(), this.getContainingTask().getEnvironment().getIOManager(), inputSerializer, binarySerializer, computer, comparator, getContainingTask().getJobConfiguration());
    this.sorter.startThreads();
    collector = new StreamRecordCollector<>(output);
    // register the metrics.
    getMetricGroup().gauge("memoryUsedSizeInBytes", (Gauge<Long>) sorter::getUsedMemoryInBytes);
    getMetricGroup().gauge("numSpillFiles", (Gauge<Long>) sorter::getNumSpillFiles);
    getMetricGroup().gauge("spillInBytes", (Gauge<Long>) sorter::getSpillInBytes);
}

Example 8 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class BinaryHashTableTest method setup.

@Before
public void setup() {
    TypeInformation[] types = new TypeInformation[] { Types.INT, Types.INT };
    this.buildSideSerializer = new BinaryRowDataSerializer(types.length);
    this.probeSideSerializer = new BinaryRowDataSerializer(types.length);
    this.ioManager = new IOManagerAsync();
    conf = new Configuration();
    conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}

Example 9 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class LongHashTableTest method init.

@Before
public void init() {
    TypeInformation[] types = new TypeInformation[] { Types.INT, Types.INT };
    this.buildSideSerializer = new BinaryRowDataSerializer(types.length);
    this.probeSideSerializer = new BinaryRowDataSerializer(types.length);
    this.ioManager = new IOManagerAsync();
    conf = new Configuration();
    conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}

Example 10 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class BinaryRowDataTest method testHashAndCopy.

@Test
public void testHashAndCopy() throws IOException {
    MemorySegment[] segments = new MemorySegment[3];
    for (int i = 0; i < 3; i++) {
        segments[i] = MemorySegmentFactory.wrap(new byte[64]);
    }
    RandomAccessOutputView out = new RandomAccessOutputView(segments, 64);
    BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(2);
    BinaryRowData row = new BinaryRowData(2);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    writer.writeString(0, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahah"));
    writer.writeString(1, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahaa"));
    writer.complete();
    serializer.serializeToPages(row, out);
    ArrayList<MemorySegment> segmentList = new ArrayList<>(Arrays.asList(segments));
    RandomAccessInputView input = new RandomAccessInputView(segmentList, 64, 64);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, input);
    assertEquals(row, mapRow);
    assertEquals(row.getString(0), mapRow.getString(0));
    assertEquals(row.getString(1), mapRow.getString(1));
    assertNotEquals(row.getString(0), mapRow.getString(1));
    // test if the hash code before and after serialization are the same
    assertEquals(row.hashCode(), mapRow.hashCode());
    assertEquals(row.getString(0).hashCode(), mapRow.getString(0).hashCode());
    assertEquals(row.getString(1).hashCode(), mapRow.getString(1).hashCode());
    // test if the copy method produce a row with the same contents
    assertEquals(row.copy(), mapRow.copy());
    assertEquals(((BinaryStringData) row.getString(0)).copy(), ((BinaryStringData) mapRow.getString(0)).copy());
    assertEquals(((BinaryStringData) row.getString(1)).copy(), ((BinaryStringData) mapRow.getString(1)).copy());
}