Search in sources :

Example 1 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class BinaryExternalSorterTest method beforeTest.

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
    this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
    this.serializer = new BinaryRowDataSerializer(2);
    this.conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 128);
}
Also used : BinaryRowDataSerializer(org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer) Before(org.junit.Before)

Example 2 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class BufferedKVExternalSorterTest method beforeTest.

@Before
public void beforeTest() throws InstantiationException, IllegalAccessException {
    this.ioManager = new IOManagerAsync();
    this.keySerializer = new BinaryRowDataSerializer(2);
    this.valueSerializer = new BinaryRowDataSerializer(2);
    this.computer = IntNormalizedKeyComputer.INSTANCE;
    this.comparator = IntRecordComparator.INSTANCE;
}
Also used : IOManagerAsync(org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync) BinaryRowDataSerializer(org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer) Before(org.junit.Before)

Example 3 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class SortMergeJoinIteratorTest method oneSideOuter.

public void oneSideOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeOneSideOuterJoinIterator iterator = new SortMergeOneSideOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            RowData probe = iterator.getProbeRow();
            if (iterator.matchKey == null) {
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(null, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, null));
                }
            } else {
                ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, probe));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}
Also used : ResettableExternalBuffer(org.apache.flink.table.runtime.util.ResettableExternalBuffer) MutableObjectIterator(org.apache.flink.util.MutableObjectIterator) IntRecordComparator(org.apache.flink.table.runtime.operators.sort.IntRecordComparator) LazyMemorySegmentPool(org.apache.flink.table.runtime.util.LazyMemorySegmentPool) RowData(org.apache.flink.table.data.RowData) BinaryRowData(org.apache.flink.table.data.binary.BinaryRowData) MyProjection(org.apache.flink.table.runtime.operators.join.Int2HashJoinOperatorTest.MyProjection) BinaryRowData(org.apache.flink.table.data.binary.BinaryRowData) BinaryRowDataSerializer(org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer)

Example 4 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class SortMergeJoinIteratorTest method inner.

public void inner(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeInnerJoinIterator iterator = new SortMergeInnerJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextInnerJoin()) {
            RowData probe = iterator.getProbeRow();
            ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
            while (iter.advanceNext()) {
                RowData row = iter.getRow();
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(row, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, row));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}
Also used : ResettableExternalBuffer(org.apache.flink.table.runtime.util.ResettableExternalBuffer) MutableObjectIterator(org.apache.flink.util.MutableObjectIterator) IntRecordComparator(org.apache.flink.table.runtime.operators.sort.IntRecordComparator) LazyMemorySegmentPool(org.apache.flink.table.runtime.util.LazyMemorySegmentPool) RowData(org.apache.flink.table.data.RowData) BinaryRowData(org.apache.flink.table.data.binary.BinaryRowData) MyProjection(org.apache.flink.table.runtime.operators.join.Int2HashJoinOperatorTest.MyProjection) BinaryRowData(org.apache.flink.table.data.binary.BinaryRowData) BinaryRowDataSerializer(org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer)

Example 5 with BinaryRowDataSerializer

use of org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer in project flink by apache.

the class SortMergeJoinIteratorTest method before.

@Before
public void before() throws MemoryAllocationException {
    this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
    this.ioManager = new IOManagerAsync();
    this.serializer = new BinaryRowDataSerializer(1);
}
Also used : IOManagerAsync(org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync) BinaryRowDataSerializer(org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer) Before(org.junit.Before)

Aggregations

BinaryRowDataSerializer (org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer)19 BinaryRowData (org.apache.flink.table.data.binary.BinaryRowData)9 Before (org.junit.Before)7 MemorySegment (org.apache.flink.core.memory.MemorySegment)5 IOManagerAsync (org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync)5 RowData (org.apache.flink.table.data.RowData)5 Test (org.junit.Test)5 ArrayList (java.util.ArrayList)4 BinaryRowWriter (org.apache.flink.table.data.writer.BinaryRowWriter)4 IntRecordComparator (org.apache.flink.table.runtime.operators.sort.IntRecordComparator)4 Configuration (org.apache.flink.configuration.Configuration)3 RandomAccessInputView (org.apache.flink.runtime.io.disk.RandomAccessInputView)3 RandomAccessOutputView (org.apache.flink.runtime.io.disk.RandomAccessOutputView)3 GenericRowData (org.apache.flink.table.data.GenericRowData)3 MyProjection (org.apache.flink.table.runtime.operators.join.Int2HashJoinOperatorTest.MyProjection)3 LazyMemorySegmentPool (org.apache.flink.table.runtime.util.LazyMemorySegmentPool)3 ResettableExternalBuffer (org.apache.flink.table.runtime.util.ResettableExternalBuffer)3 Random (java.util.Random)2 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)2 JoinedRowData (org.apache.flink.table.data.utils.JoinedRowData)2