Examples with DataFrame io.questdb.cairo.sql.DataFrame used on opensource projects

Example 6 with DataFrame

use of io.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class FullBwdDataFrameCursorTest method printCursor.

private void printCursor(TableReaderRecord record, DataFrameCursor cursor) {
    sink.clear();
    record.of(cursor.getTableReader());
    DataFrame frame;
    while ((frame = cursor.next()) != null) {
        record.jumpTo(frame.getPartitionIndex(), 0);
        for (long index = frame.getRowHi() - 1, lo = frame.getRowLo() - 1; index > lo; index--) {
            record.setRecordIndex(index);
            printer.print(record, cursor.getTableReader().getMetadata(), sink);
        }
    }
}

Example 7 with DataFrame

use of io.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class LatestByAllIndexedRecordCursor method buildTreeMap.

@Override
protected void buildTreeMap(SqlExecutionContext executionContext) throws SqlException {
    final MessageBus bus = executionContext.getMessageBus();
    final RingQueue<LatestByTask> queue = bus.getLatestByQueue();
    final Sequence pubSeq = bus.getLatestByPubSeq();
    final Sequence subSeq = bus.getLatestBySubSeq();
    int keyCount = getSymbolTable(columnIndex).size() + 1;
    rows.extend(keyCount);
    GeoHashNative.iota(rows.getAddress(), rows.getCapacity(), 0);
    final int workerCount = executionContext.getWorkerCount();
    final long chunkSize = (keyCount + workerCount - 1) / workerCount;
    final int taskCount = (int) ((keyCount + chunkSize - 1) / chunkSize);
    final long argumentsAddress = LatestByArguments.allocateMemoryArray(taskCount);
    for (long i = 0; i < taskCount; ++i) {
        final long klo = i * chunkSize;
        final long khi = Long.min(klo + chunkSize, keyCount);
        final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
        LatestByArguments.setRowsAddress(argsAddress, rows.getAddress());
        LatestByArguments.setRowsCapacity(argsAddress, rows.getCapacity());
        LatestByArguments.setKeyLo(argsAddress, klo);
        LatestByArguments.setKeyHi(argsAddress, khi);
        LatestByArguments.setRowsSize(argsAddress, 0);
    }
    int hashColumnIndex = -1;
    int hashColumnType = ColumnType.UNDEFINED;
    long prefixesAddress = 0;
    long prefixesCount = 0;
    if (this.prefixes.size() > 2) {
        hashColumnIndex = (int) prefixes.get(0);
        hashColumnType = (int) prefixes.get(1);
        prefixesAddress = prefixes.getAddress() + 2 * Long.BYTES;
        prefixesCount = prefixes.size() - 2;
    }
    DataFrame frame;
    // frame metadata is based on TableReader, which is "full" metadata
    // this cursor works with subset of columns, which warrants column index remap
    int frameColumnIndex = columnIndexes.getQuick(columnIndex);
    final TableReader reader = this.dataFrameCursor.getTableReader();
    long foundRowCount = 0;
    while ((frame = this.dataFrameCursor.next()) != null && foundRowCount < keyCount) {
        doneLatch.reset();
        final BitmapIndexReader indexReader = frame.getBitmapIndexReader(frameColumnIndex, BitmapIndexReader.DIR_BACKWARD);
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        final long keyBaseAddress = indexReader.getKeyBaseAddress();
        final long keysMemorySize = indexReader.getKeyMemorySize();
        final long valueBaseAddress = indexReader.getValueBaseAddress();
        final long valuesMemorySize = indexReader.getValueMemorySize();
        final int valueBlockCapacity = indexReader.getValueBlockCapacity();
        final long unIndexedNullCount = indexReader.getUnIndexedNullCount();
        final int partitionIndex = frame.getPartitionIndex();
        long hashColumnAddress = 0;
        // hashColumnIndex can be -1 for latest by part only (no prefixes to match)
        if (hashColumnIndex > -1) {
            final int columnBase = reader.getColumnBase(partitionIndex);
            final int primaryColumnIndex = TableReader.getPrimaryColumnIndex(columnBase, hashColumnIndex);
            final MemoryR column = reader.getColumn(primaryColumnIndex);
            hashColumnAddress = column.getPageAddress(0);
        }
        // -1 must be dead case here
        final int hashesColumnSize = ColumnType.isGeoHash(hashColumnType) ? getPow2SizeOfGeoHashType(hashColumnType) : -1;
        int queuedCount = 0;
        for (long i = 0; i < taskCount; ++i) {
            final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            final long found = LatestByArguments.getRowsSize(argsAddress);
            final long keyHi = LatestByArguments.getKeyHi(argsAddress);
            final long keyLo = LatestByArguments.getKeyLo(argsAddress);
            // Skip range if all keys found
            if (found >= keyHi - keyLo) {
                continue;
            }
            // Update hash column address with current frame value
            LatestByArguments.setHashesAddress(argsAddress, hashColumnAddress);
            final long seq = pubSeq.next();
            if (seq < 0) {
                GeoHashNative.latestByAndFilterPrefix(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount);
            } else {
                queue.get(seq).of(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount, doneLatch);
                pubSeq.done(seq);
                queuedCount++;
            }
        }
        // this should fix deadlock with 1 worker configuration
        while (doneLatch.getCount() > -queuedCount) {
            long seq = subSeq.next();
            if (seq > -1) {
                queue.get(seq).run();
                subSeq.done(seq);
            }
        }
        doneLatch.await(queuedCount);
        // Reset found counter
        foundRowCount = 0;
        for (int i = 0; i < taskCount; i++) {
            final long address = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            foundRowCount += LatestByArguments.getRowsSize(address);
        }
    }
    final long rowCount = GeoHashNative.slideFoundBlocks(argumentsAddress, taskCount);
    LatestByArguments.releaseMemoryArray(argumentsAddress, taskCount);
    aLimit = rowCount;
    aIndex = indexShift;
    postProcessRows();
}

Example 8 with DataFrame

use of io.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class LatestByValuesFilteredRecordCursor method buildTreeMap.

@Override
protected void buildTreeMap(SqlExecutionContext executionContext) {
    prepare();
    DataFrame frame;
    while ((frame = this.dataFrameCursor.next()) != null) {
        final int partitionIndex = frame.getPartitionIndex();
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        recordA.jumpTo(frame.getPartitionIndex(), rowHi);
        for (long row = rowHi; row >= rowLo; row--) {
            recordA.setRecordIndex(row);
            if (filter.getBool(recordA)) {
                int key = TableUtils.toIndexKey(recordA.getInt(columnIndex));
                int index = map.keyIndex(key);
                if (index < 0 && map.valueAt(index) == 0) {
                    rows.add(Rows.toRowID(partitionIndex, row));
                    map.putAt(index, key, 1);
                }
            }
        }
    }
}

Example 9 with DataFrame

use of io.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class LatestByValuesIndexedFilteredRecordCursor method buildTreeMap.

@Override
protected void buildTreeMap(SqlExecutionContext executionContext) {
    final int keyCount = symbolKeys.size();
    found.clear();
    DataFrame frame;
    while ((frame = this.dataFrameCursor.next()) != null && found.size() < keyCount) {
        final int partitionIndex = frame.getPartitionIndex();
        final BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex, BitmapIndexReader.DIR_BACKWARD);
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        this.recordA.jumpTo(partitionIndex, 0);
        for (int i = 0, n = symbolKeys.size(); i < n; i++) {
            int symbolKey = symbolKeys.get(i);
            int index = found.keyIndex(symbolKey);
            if (index > -1) {
                RowCursor cursor = indexReader.getCursor(false, symbolKey, rowLo, rowHi);
                if (cursor.hasNext()) {
                    final long row = cursor.next();
                    recordA.setRecordIndex(row);
                    if (filter.getBool(recordA)) {
                        rows.add(Rows.toRowID(partitionIndex, row));
                        found.addAt(index, symbolKey);
                    }
                }
            }
        }
    }
    rows.sortAsUnsigned();
}

Example 10 with DataFrame

use of io.questdb.cairo.sql.DataFrame in project questdb by bluestreak01.

the class LatestByValuesIndexedRecordCursor method buildTreeMap.

protected void buildTreeMap() {
    final int keyCount = symbolKeys.size();
    found.clear();
    rows.setPos(0);
    DataFrame frame;
    while ((frame = this.dataFrameCursor.next()) != null && found.size() < keyCount) {
        final BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex, BitmapIndexReader.DIR_BACKWARD);
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        for (int i = 0, n = symbolKeys.size(); i < n; i++) {
            int symbolKey = symbolKeys.get(i);
            int index = found.keyIndex(symbolKey);
            if (index > -1) {
                RowCursor cursor = indexReader.getCursor(false, symbolKey, rowLo, rowHi);
                if (cursor.hasNext()) {
                    final long row = Rows.toRowID(frame.getPartitionIndex(), cursor.next());
                    rows.add(row);
                    found.addAt(index, symbolKey);
                }
            }
        }
    }
    index = rows.size() - 1;
}