Examples with LatestByTask io.questdb.tasks.LatestByTask used on opensource projects

Example 1 with LatestByTask

use of io.questdb.tasks.LatestByTask in project questdb by bluestreak01.

the class LatestByAllIndexedRecordCursor method buildTreeMap.

@Override
protected void buildTreeMap(SqlExecutionContext executionContext) throws SqlException {
    final MessageBus bus = executionContext.getMessageBus();
    final RingQueue<LatestByTask> queue = bus.getLatestByQueue();
    final Sequence pubSeq = bus.getLatestByPubSeq();
    final Sequence subSeq = bus.getLatestBySubSeq();
    int keyCount = getSymbolTable(columnIndex).size() + 1;
    rows.extend(keyCount);
    GeoHashNative.iota(rows.getAddress(), rows.getCapacity(), 0);
    final int workerCount = executionContext.getWorkerCount();
    final long chunkSize = (keyCount + workerCount - 1) / workerCount;
    final int taskCount = (int) ((keyCount + chunkSize - 1) / chunkSize);
    final long argumentsAddress = LatestByArguments.allocateMemoryArray(taskCount);
    for (long i = 0; i < taskCount; ++i) {
        final long klo = i * chunkSize;
        final long khi = Long.min(klo + chunkSize, keyCount);
        final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
        LatestByArguments.setRowsAddress(argsAddress, rows.getAddress());
        LatestByArguments.setRowsCapacity(argsAddress, rows.getCapacity());
        LatestByArguments.setKeyLo(argsAddress, klo);
        LatestByArguments.setKeyHi(argsAddress, khi);
        LatestByArguments.setRowsSize(argsAddress, 0);
    }
    int hashColumnIndex = -1;
    int hashColumnType = ColumnType.UNDEFINED;
    long prefixesAddress = 0;
    long prefixesCount = 0;
    if (this.prefixes.size() > 2) {
        hashColumnIndex = (int) prefixes.get(0);
        hashColumnType = (int) prefixes.get(1);
        prefixesAddress = prefixes.getAddress() + 2 * Long.BYTES;
        prefixesCount = prefixes.size() - 2;
    }
    DataFrame frame;
    // frame metadata is based on TableReader, which is "full" metadata
    // this cursor works with subset of columns, which warrants column index remap
    int frameColumnIndex = columnIndexes.getQuick(columnIndex);
    final TableReader reader = this.dataFrameCursor.getTableReader();
    long foundRowCount = 0;
    while ((frame = this.dataFrameCursor.next()) != null && foundRowCount < keyCount) {
        doneLatch.reset();
        final BitmapIndexReader indexReader = frame.getBitmapIndexReader(frameColumnIndex, BitmapIndexReader.DIR_BACKWARD);
        final long rowLo = frame.getRowLo();
        final long rowHi = frame.getRowHi() - 1;
        final long keyBaseAddress = indexReader.getKeyBaseAddress();
        final long keysMemorySize = indexReader.getKeyMemorySize();
        final long valueBaseAddress = indexReader.getValueBaseAddress();
        final long valuesMemorySize = indexReader.getValueMemorySize();
        final int valueBlockCapacity = indexReader.getValueBlockCapacity();
        final long unIndexedNullCount = indexReader.getUnIndexedNullCount();
        final int partitionIndex = frame.getPartitionIndex();
        long hashColumnAddress = 0;
        // hashColumnIndex can be -1 for latest by part only (no prefixes to match)
        if (hashColumnIndex > -1) {
            final int columnBase = reader.getColumnBase(partitionIndex);
            final int primaryColumnIndex = TableReader.getPrimaryColumnIndex(columnBase, hashColumnIndex);
            final MemoryR column = reader.getColumn(primaryColumnIndex);
            hashColumnAddress = column.getPageAddress(0);
        }
        // -1 must be dead case here
        final int hashesColumnSize = ColumnType.isGeoHash(hashColumnType) ? getPow2SizeOfGeoHashType(hashColumnType) : -1;
        int queuedCount = 0;
        for (long i = 0; i < taskCount; ++i) {
            final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            final long found = LatestByArguments.getRowsSize(argsAddress);
            final long keyHi = LatestByArguments.getKeyHi(argsAddress);
            final long keyLo = LatestByArguments.getKeyLo(argsAddress);
            // Skip range if all keys found
            if (found >= keyHi - keyLo) {
                continue;
            }
            // Update hash column address with current frame value
            LatestByArguments.setHashesAddress(argsAddress, hashColumnAddress);
            final long seq = pubSeq.next();
            if (seq < 0) {
                GeoHashNative.latestByAndFilterPrefix(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount);
            } else {
                queue.get(seq).of(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount, doneLatch);
                pubSeq.done(seq);
                queuedCount++;
            }
        }
        // this should fix deadlock with 1 worker configuration
        while (doneLatch.getCount() > -queuedCount) {
            long seq = subSeq.next();
            if (seq > -1) {
                queue.get(seq).run();
                subSeq.done(seq);
            }
        }
        doneLatch.await(queuedCount);
        // Reset found counter
        foundRowCount = 0;
        for (int i = 0; i < taskCount; i++) {
            final long address = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
            foundRowCount += LatestByArguments.getRowsSize(address);
        }
    }
    final long rowCount = GeoHashNative.slideFoundBlocks(argumentsAddress, taskCount);
    LatestByArguments.releaseMemoryArray(argumentsAddress, taskCount);
    aLimit = rowCount;
    aIndex = indexShift;
    postProcessRows();
}

Example 2 with LatestByTask

use of io.questdb.tasks.LatestByTask in project questdb by bluestreak01.

the class LatestByAllIndexedJob method doRun.

@Override
protected boolean doRun(int workerId, long cursor) {
    final LatestByTask task = queue.get(cursor);
    final boolean result = task.run();
    subSeq.done(cursor);
    return result;
}