Examples with BinaryRowData org.apache.flink.table.data.binary.BinaryRowData used on opensource projects

Example 16 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryExternalSorterTest method testSortTwoBufferInMemory.

@Test
public void testSortTwoBufferInMemory() throws Exception {
    int size = 1_000_000;
    MockBinaryRowReader reader = new MockBinaryRowReader(size);
    LOG.debug("initializing sortmerger");
    // there are two sort buffer if sortMemory > 100 * 1024 * 1024.
    MemoryManager memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024 * 101).build();
    long minMemorySize = memoryManager.computeNumberOfPages(1) * MemoryManager.DEFAULT_PAGE_SIZE;
    BinaryExternalSorter sorter = new BinaryExternalSorter(new Object(), memoryManager, minMemorySize, this.ioManager, (AbstractRowDataSerializer) serializer, serializer, IntNormalizedKeyComputer.INSTANCE, IntRecordComparator.INSTANCE, conf, 1f);
    sorter.startThreads();
    sorter.write(reader);
    MutableObjectIterator<BinaryRowData> iterator = sorter.getIterator();
    BinaryRowData next = serializer.createInstance();
    for (int i = 0; i < size; i++) {
        next = iterator.next(next);
        Assert.assertEquals(i, next.getInt(0));
        Assert.assertEquals(getString(i), next.getString(1).toString());
    }
    sorter.close();
    Assert.assertTrue(memoryManager.verifyEmpty());
    memoryManager.shutdown();
}

Example 17 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryExternalSorterTest method testSpilling.

@Test
public void testSpilling() throws Exception {
    int size = 1000_000;
    MockBinaryRowReader reader = new MockBinaryRowReader(size);
    LOG.debug("initializing sortmerger");
    long minMemorySize = memoryManager.computeNumberOfPages(0.1) * MemoryManager.DEFAULT_PAGE_SIZE;
    BinaryExternalSorter sorter = new BinaryExternalSorter(new Object(), this.memoryManager, minMemorySize, this.ioManager, (AbstractRowDataSerializer) serializer, serializer, IntNormalizedKeyComputer.INSTANCE, IntRecordComparator.INSTANCE, conf, 0.7f);
    sorter.startThreads();
    sorter.write(reader);
    MutableObjectIterator<BinaryRowData> iterator = sorter.getIterator();
    BinaryRowData next = serializer.createInstance();
    for (int i = 0; i < size; i++) {
        next = iterator.next(next);
        Assert.assertEquals(i, next.getInt(0));
        Assert.assertEquals(getString(i), next.getString(1).toString());
    }
    sorter.close();
}

Example 18 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryExternalSorterTest method testSort.

@Test
public void testSort() throws Exception {
    int size = 10_000;
    MockBinaryRowReader reader = new MockBinaryRowReader(size);
    LOG.debug("initializing sortmerger");
    long minMemorySize = memoryManager.computeNumberOfPages(0.9) * MemoryManager.DEFAULT_PAGE_SIZE;
    BinaryExternalSorter sorter = new BinaryExternalSorter(new Object(), this.memoryManager, minMemorySize, this.ioManager, (AbstractRowDataSerializer) serializer, serializer, IntNormalizedKeyComputer.INSTANCE, IntRecordComparator.INSTANCE, conf, 0.7f);
    sorter.startThreads();
    sorter.write(reader);
    MutableObjectIterator<BinaryRowData> iterator = sorter.getIterator();
    BinaryRowData next = serializer.createInstance();
    for (int i = 0; i < size; i++) {
        next = iterator.next(next);
        Assert.assertEquals(i, next.getInt(0));
        Assert.assertEquals(getString(i), next.getString(1).toString());
    }
    sorter.close();
}

Example 19 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class StreamSortOperatorTest method test.

@Test
public void test() throws Exception {
    StreamSortOperator operator = createSortOperator();
    OneInputStreamOperatorTestHarness<RowData, BinaryRowData> testHarness = createTestHarness(operator);
    testHarness.open();
    testHarness.processElement(insertRecord("hi", 1));
    testHarness.processElement(insertRecord("hello", 2));
    testHarness.processElement(insertRecord("world", 3));
    testHarness.processElement(insertRecord("word", 4));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord("hello", 2));
    expectedOutput.add(insertRecord("hi", 1));
    expectedOutput.add(insertRecord("word", 4));
    expectedOutput.add(insertRecord("world", 3));
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    testHarness.close();
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    expectedOutput.clear();
    operator = createSortOperator();
    testHarness = createTestHarness(operator);
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processElement(insertRecord("abc", 1));
    testHarness.processElement(insertRecord("aa", 1));
    testHarness.close();
    expectedOutput.add(insertRecord("aa", 1));
    expectedOutput.add(insertRecord("abc", 1));
    expectedOutput.add(insertRecord("hello", 2));
    expectedOutput.add(insertRecord("hi", 1));
    expectedOutput.add(insertRecord("word", 4));
    expectedOutput.add(insertRecord("world", 3));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}

Example 20 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class SortMergeJoinIteratorTest method oneSideOuter.

public void oneSideOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeOneSideOuterJoinIterator iterator = new SortMergeOneSideOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            RowData probe = iterator.getProbeRow();
            if (iterator.matchKey == null) {
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(null, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, null));
                }
            } else {
                ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, probe));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}