Examples with ArrowReader org.apache.flink.table.runtime.arrow.ArrowReader used on opensource projects

Example 1 with ArrowReader

use of org.apache.flink.table.runtime.arrow.ArrowReader in project flink by apache.

the class ArrowSourceFunction method run.

@Override
public void run(SourceContext<RowData> ctx) throws Exception {
    VectorLoader vectorLoader = new VectorLoader(root);
    while (running && !indexesToEmit.isEmpty()) {
        Tuple2<Integer, Integer> indexToEmit = indexesToEmit.peek();
        ArrowRecordBatch arrowRecordBatch = loadBatch(indexToEmit.f0);
        vectorLoader.load(arrowRecordBatch);
        arrowRecordBatch.close();
        ArrowReader arrowReader = createArrowReader(root);
        int rowCount = root.getRowCount();
        int nextRowId = indexToEmit.f1;
        while (nextRowId < rowCount) {
            RowData element = arrowReader.read(nextRowId);
            synchronized (ctx.getCheckpointLock()) {
                ctx.collect(element);
                indexToEmit.setField(++nextRowId, 1);
            }
        }
        synchronized (ctx.getCheckpointLock()) {
            indexesToEmit.pop();
        }
    }
}