Examples with BinaryRowData org.apache.flink.table.data.binary.BinaryRowData used on opensource projects

Example 11 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryExternalSorterTest method testMergeManyTimes.

@Test
public void testMergeManyTimes() throws Exception {
    int size = 1000_000;
    MockBinaryRowReader reader = new MockBinaryRowReader(size);
    LOG.debug("initializing sortmerger");
    long minMemorySize = memoryManager.computeNumberOfPages(0.01) * MemoryManager.DEFAULT_PAGE_SIZE;
    conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 8);
    BinaryExternalSorter sorter = new BinaryExternalSorter(new Object(), this.memoryManager, minMemorySize, this.ioManager, (AbstractRowDataSerializer) serializer, serializer, IntNormalizedKeyComputer.INSTANCE, IntRecordComparator.INSTANCE, conf, 0.7f);
    sorter.startThreads();
    sorter.write(reader);
    MutableObjectIterator<BinaryRowData> iterator = sorter.getIterator();
    BinaryRowData next = serializer.createInstance();
    for (int i = 0; i < size; i++) {
        next = iterator.next(next);
        Assert.assertEquals(i, next.getInt(0));
        Assert.assertEquals(getString(i), next.getString(1).toString());
    }
    sorter.close();
}

Example 12 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryExternalSorterTest method testSpillingDesc.

@Test
public void testSpillingDesc() throws Exception {
    int size = 1000_000;
    MockBinaryRowReader reader = new MockBinaryRowReader(size);
    LOG.debug("initializing sortmerger");
    long minMemorySize = memoryManager.computeNumberOfPages(0.1) * MemoryManager.DEFAULT_PAGE_SIZE;
    BinaryExternalSorter sorter = new BinaryExternalSorter(new Object(), this.memoryManager, minMemorySize, this.ioManager, (AbstractRowDataSerializer) serializer, serializer, new IntNormalizedKeyComputer() {

        @Override
        public boolean invertKey() {
            return true;
        }
    }, new IntRecordComparator() {

        @Override
        public int compare(RowData o1, RowData o2) {
            return -super.compare(o1, o2);
        }
    }, conf, 0.7f);
    sorter.startThreads();
    sorter.write(reader);
    MutableObjectIterator<BinaryRowData> iterator = sorter.getIterator();
    List<Tuple2<Integer, String>> data = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        data.add(new Tuple2<>(i, getString(i)));
    }
    data.sort((o1, o2) -> -o1.f0.compareTo(o2.f0));
    BinaryRowData next = serializer.createInstance();
    for (int i = 0; i < size; i++) {
        next = iterator.next(next);
        Assert.assertEquals((int) data.get(i).f0, next.getInt(0));
        Assert.assertEquals(data.get(i).f1, next.getString(1).toString());
    }
    sorter.close();
}

Example 13 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryMergeIteratorTest method newIterator.

private MutableObjectIterator<BinaryRowData> newIterator(final int[] keys, final String[] values) {
    BinaryRowData row = serializer.createInstance();
    BinaryRowWriter writer = new BinaryRowWriter(row);
    return new MutableObjectIterator<BinaryRowData>() {

        private int current = 0;

        @Override
        public BinaryRowData next(BinaryRowData reuse) {
            if (current < keys.length) {
                int key = keys[current];
                String value = values[current];
                current++;
                writer.reset();
                writer.writeInt(0, key);
                writer.writeString(1, StringData.fromString(value));
                writer.complete();
                return row;
            } else {
                return null;
            }
        }

        @Override
        public BinaryRowData next() {
            throw new RuntimeException();
        }
    };
}

Example 14 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryMergeIteratorTest method testMergeOfTenStreams.

@Test
public void testMergeOfTenStreams() throws Exception {
    List<MutableObjectIterator<BinaryRowData>> iterators = new ArrayList<>();
    iterators.add(newIterator(new int[] { 1, 2, 17, 23, 23 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 2, 6, 7, 8, 9 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 4, 10, 11, 11, 12 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 3, 6, 7, 10, 12 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 7, 10, 15, 19, 44 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 6, 6, 11, 17, 18 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 1, 2, 4, 5, 10 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 5, 10, 19, 23, 29 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 9, 9, 9, 9, 9 }, new String[] { "A", "B", "C", "D", "E" }));
    iterators.add(newIterator(new int[] { 8, 8, 14, 14, 15 }, new String[] { "A", "B", "C", "D", "E" }));
    TypeComparator<Integer> comparator = new IntComparator(true);
    MutableObjectIterator<BinaryRowData> iterator = new BinaryMergeIterator<>(iterators, reused(10), (o1, o2) -> this.comparator.compare(o1, o2));
    BinaryRowData row = serializer.createInstance();
    int pre = 0;
    while ((row = iterator.next(row)) != null) {
        Assert.assertTrue(comparator.compare(row.getInt(0), pre) >= 0);
        pre = row.getInt(0);
    }
}

Example 15 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class BinaryMergeIteratorTest method testMergeOfTwoStreams.

@Test
public void testMergeOfTwoStreams() throws Exception {
    List<MutableObjectIterator<BinaryRowData>> iterators = new ArrayList<>();
    iterators.add(newIterator(new int[] { 1, 2, 4, 5, 10 }, new String[] { "1", "2", "4", "5", "10" }));
    iterators.add(newIterator(new int[] { 3, 6, 7, 10, 12 }, new String[] { "3", "6", "7", "10", "12" }));
    final int[] expected = new int[] { 1, 2, 3, 4, 5, 6, 7, 10, 10, 12 };
    MutableObjectIterator<BinaryRowData> iterator = new BinaryMergeIterator<>(iterators, reused(2), (o1, o2) -> this.comparator.compare(o1, o2));
    BinaryRowData row = serializer.createInstance();
    int pos = 0;
    while ((row = iterator.next(row)) != null) {
        Assert.assertEquals(expected[pos++], row.getInt(0));
    }
}