Examples with ResettableExternalBuffer org.apache.flink.table.runtime.util.ResettableExternalBuffer used on opensource projects

Example 1 with ResettableExternalBuffer

use of org.apache.flink.table.runtime.util.ResettableExternalBuffer in project flink by apache.

the class SortMergeJoinIteratorTest method oneSideOuter.

public void oneSideOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeOneSideOuterJoinIterator iterator = new SortMergeOneSideOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            RowData probe = iterator.getProbeRow();
            if (iterator.matchKey == null) {
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(null, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, null));
                }
            } else {
                ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, probe));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 2 with ResettableExternalBuffer

use of org.apache.flink.table.runtime.util.ResettableExternalBuffer in project flink by apache.

the class SortMergeJoinIteratorTest method inner.

public void inner(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeInnerJoinIterator iterator = new SortMergeInnerJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextInnerJoin()) {
            RowData probe = iterator.getProbeRow();
            ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
            while (iter.advanceNext()) {
                RowData row = iter.getRow();
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(row, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, row));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 3 with ResettableExternalBuffer

use of org.apache.flink.table.runtime.util.ResettableExternalBuffer in project flink by apache.

the class SortMergeJoinIteratorTest method fullOuter.

public void fullOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator<BinaryRowData> input1 = data.f0;
    MutableObjectIterator<BinaryRowData> input2 = data.f1;
    try (SortMergeFullOuterJoinIterator iterator = new SortMergeFullOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            BinaryRowData matchKey = iterator.getMatchKey();
            ResettableExternalBuffer buffer1 = iterator.getBuffer1();
            ResettableExternalBuffer buffer2 = iterator.getBuffer2();
            if (matchKey == null && buffer1.size() > 0) {
                // left outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer1.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, null));
                }
            } else if (matchKey == null && buffer2.size() > 0) {
                // right outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer2.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(null, row));
                }
            } else if (matchKey != null) {
                // match join.
                ResettableExternalBuffer.BufferIterator iter1 = buffer1.newIterator();
                while (iter1.advanceNext()) {
                    RowData row1 = iter1.getRow();
                    ResettableExternalBuffer.BufferIterator iter2 = buffer2.newIterator();
                    while (iter2.advanceNext()) {
                        RowData row2 = iter2.getRow();
                        Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                        assertEquals(expected, new Tuple2<>(row1, row2));
                    }
                }
            } else {
                // bug...
                throw new RuntimeException("There is a bug.");
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 4 with ResettableExternalBuffer

use of org.apache.flink.table.runtime.util.ResettableExternalBuffer in project flink by apache.

the class BufferDataOverWindowOperator method open.

@Override
public void open() throws Exception {
    super.open();
    ClassLoader cl = getUserCodeClassloader();
    serializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(cl);
    partitionComparator = genComparator.newInstance(cl);
    genComparator = null;
    MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
    LazyMemorySegmentPool pool = new LazyMemorySegmentPool(this, memManager, (int) (computeMemorySize() / memManager.getPageSize()));
    this.currentData = new ResettableExternalBuffer(getContainingTask().getEnvironment().getIOManager(), pool, serializer, isRowAllInFixedPart);
    collector = new StreamRecordCollector<>(output);
    joinedRows = new JoinedRowData[overWindowFrames.length];
    for (int i = 0; i < overWindowFrames.length; i++) {
        overWindowFrames[i].open(new ExecutionContextImpl(this, getRuntimeContext()));
        joinedRows[i] = new JoinedRowData();
    }
}

Example 5 with ResettableExternalBuffer

use of org.apache.flink.table.runtime.util.ResettableExternalBuffer in project flink by apache.

the class SortMergeJoinOperator method fullOuterJoin.

private void fullOuterJoin(SortMergeFullOuterJoinIterator iterator) throws Exception {
    BitSet bitSet = new BitSet();
    while (iterator.nextOuterJoin()) {
        bitSet.clear();
        BinaryRowData matchKey = iterator.getMatchKey();
        ResettableExternalBuffer buffer1 = iterator.getBuffer1();
        ResettableExternalBuffer buffer2 = iterator.getBuffer2();
        if (matchKey == null && buffer1.size() > 0) {
            // left outer join.
            ResettableExternalBuffer.BufferIterator iter = buffer1.newIterator();
            while (iter.advanceNext()) {
                RowData row1 = iter.getRow();
                collector.collect(joinedRow.replace(row1, rightNullRow));
            }
            iter.close();
        } else if (matchKey == null && buffer2.size() > 0) {
            // right outer join.
            ResettableExternalBuffer.BufferIterator iter = buffer2.newIterator();
            while (iter.advanceNext()) {
                RowData row2 = iter.getRow();
                collector.collect(joinedRow.replace(leftNullRow, row2));
            }
            iter.close();
        } else if (matchKey != null) {
            // match join.
            ResettableExternalBuffer.BufferIterator iter1 = buffer1.newIterator();
            while (iter1.advanceNext()) {
                RowData row1 = iter1.getRow();
                boolean found = false;
                int index = 0;
                ResettableExternalBuffer.BufferIterator iter2 = buffer2.newIterator();
                while (iter2.advanceNext()) {
                    RowData row2 = iter2.getRow();
                    if (condFunc.apply(row1, row2)) {
                        collector.collect(joinedRow.replace(row1, row2));
                        found = true;
                        bitSet.set(index);
                    }
                    index++;
                }
                iter2.close();
                if (!found) {
                    collector.collect(joinedRow.replace(row1, rightNullRow));
                }
            }
            iter1.close();
            // row2 outer
            int index = 0;
            ResettableExternalBuffer.BufferIterator iter2 = buffer2.newIterator();
            while (iter2.advanceNext()) {
                RowData row2 = iter2.getRow();
                if (!bitSet.get(index)) {
                    collector.collect(joinedRow.replace(leftNullRow, row2));
                }
                index++;
            }
            iter2.close();
        } else {
            // bug...
            throw new RuntimeException("There is a bug.");
        }
    }
}