use of io.questdb.cairo.BitmapIndexReader in project questdb by bluestreak01.
the class LatestByAllIndexedRecordCursor method buildTreeMap.
@Override
protected void buildTreeMap(SqlExecutionContext executionContext) throws SqlException {
final MessageBus bus = executionContext.getMessageBus();
final RingQueue<LatestByTask> queue = bus.getLatestByQueue();
final Sequence pubSeq = bus.getLatestByPubSeq();
final Sequence subSeq = bus.getLatestBySubSeq();
int keyCount = getSymbolTable(columnIndex).size() + 1;
rows.extend(keyCount);
GeoHashNative.iota(rows.getAddress(), rows.getCapacity(), 0);
final int workerCount = executionContext.getWorkerCount();
final long chunkSize = (keyCount + workerCount - 1) / workerCount;
final int taskCount = (int) ((keyCount + chunkSize - 1) / chunkSize);
final long argumentsAddress = LatestByArguments.allocateMemoryArray(taskCount);
for (long i = 0; i < taskCount; ++i) {
final long klo = i * chunkSize;
final long khi = Long.min(klo + chunkSize, keyCount);
final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
LatestByArguments.setRowsAddress(argsAddress, rows.getAddress());
LatestByArguments.setRowsCapacity(argsAddress, rows.getCapacity());
LatestByArguments.setKeyLo(argsAddress, klo);
LatestByArguments.setKeyHi(argsAddress, khi);
LatestByArguments.setRowsSize(argsAddress, 0);
}
int hashColumnIndex = -1;
int hashColumnType = ColumnType.UNDEFINED;
long prefixesAddress = 0;
long prefixesCount = 0;
if (this.prefixes.size() > 2) {
hashColumnIndex = (int) prefixes.get(0);
hashColumnType = (int) prefixes.get(1);
prefixesAddress = prefixes.getAddress() + 2 * Long.BYTES;
prefixesCount = prefixes.size() - 2;
}
DataFrame frame;
// frame metadata is based on TableReader, which is "full" metadata
// this cursor works with subset of columns, which warrants column index remap
int frameColumnIndex = columnIndexes.getQuick(columnIndex);
final TableReader reader = this.dataFrameCursor.getTableReader();
long foundRowCount = 0;
while ((frame = this.dataFrameCursor.next()) != null && foundRowCount < keyCount) {
doneLatch.reset();
final BitmapIndexReader indexReader = frame.getBitmapIndexReader(frameColumnIndex, BitmapIndexReader.DIR_BACKWARD);
final long rowLo = frame.getRowLo();
final long rowHi = frame.getRowHi() - 1;
final long keyBaseAddress = indexReader.getKeyBaseAddress();
final long keysMemorySize = indexReader.getKeyMemorySize();
final long valueBaseAddress = indexReader.getValueBaseAddress();
final long valuesMemorySize = indexReader.getValueMemorySize();
final int valueBlockCapacity = indexReader.getValueBlockCapacity();
final long unIndexedNullCount = indexReader.getUnIndexedNullCount();
final int partitionIndex = frame.getPartitionIndex();
long hashColumnAddress = 0;
// hashColumnIndex can be -1 for latest by part only (no prefixes to match)
if (hashColumnIndex > -1) {
final int columnBase = reader.getColumnBase(partitionIndex);
final int primaryColumnIndex = TableReader.getPrimaryColumnIndex(columnBase, hashColumnIndex);
final MemoryR column = reader.getColumn(primaryColumnIndex);
hashColumnAddress = column.getPageAddress(0);
}
// -1 must be dead case here
final int hashesColumnSize = ColumnType.isGeoHash(hashColumnType) ? getPow2SizeOfGeoHashType(hashColumnType) : -1;
int queuedCount = 0;
for (long i = 0; i < taskCount; ++i) {
final long argsAddress = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
final long found = LatestByArguments.getRowsSize(argsAddress);
final long keyHi = LatestByArguments.getKeyHi(argsAddress);
final long keyLo = LatestByArguments.getKeyLo(argsAddress);
// Skip range if all keys found
if (found >= keyHi - keyLo) {
continue;
}
// Update hash column address with current frame value
LatestByArguments.setHashesAddress(argsAddress, hashColumnAddress);
final long seq = pubSeq.next();
if (seq < 0) {
GeoHashNative.latestByAndFilterPrefix(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount);
} else {
queue.get(seq).of(keyBaseAddress, keysMemorySize, valueBaseAddress, valuesMemorySize, argsAddress, unIndexedNullCount, rowHi, rowLo, partitionIndex, valueBlockCapacity, hashColumnAddress, hashesColumnSize, prefixesAddress, prefixesCount, doneLatch);
pubSeq.done(seq);
queuedCount++;
}
}
// this should fix deadlock with 1 worker configuration
while (doneLatch.getCount() > -queuedCount) {
long seq = subSeq.next();
if (seq > -1) {
queue.get(seq).run();
subSeq.done(seq);
}
}
doneLatch.await(queuedCount);
// Reset found counter
foundRowCount = 0;
for (int i = 0; i < taskCount; i++) {
final long address = argumentsAddress + i * LatestByArguments.MEMORY_SIZE;
foundRowCount += LatestByArguments.getRowsSize(address);
}
}
final long rowCount = GeoHashNative.slideFoundBlocks(argumentsAddress, taskCount);
LatestByArguments.releaseMemoryArray(argumentsAddress, taskCount);
aLimit = rowCount;
aIndex = indexShift;
postProcessRows();
}
use of io.questdb.cairo.BitmapIndexReader in project questdb by bluestreak01.
the class LatestByValuesIndexedFilteredRecordCursor method buildTreeMap.
@Override
protected void buildTreeMap(SqlExecutionContext executionContext) {
final int keyCount = symbolKeys.size();
found.clear();
DataFrame frame;
while ((frame = this.dataFrameCursor.next()) != null && found.size() < keyCount) {
final int partitionIndex = frame.getPartitionIndex();
final BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex, BitmapIndexReader.DIR_BACKWARD);
final long rowLo = frame.getRowLo();
final long rowHi = frame.getRowHi() - 1;
this.recordA.jumpTo(partitionIndex, 0);
for (int i = 0, n = symbolKeys.size(); i < n; i++) {
int symbolKey = symbolKeys.get(i);
int index = found.keyIndex(symbolKey);
if (index > -1) {
RowCursor cursor = indexReader.getCursor(false, symbolKey, rowLo, rowHi);
if (cursor.hasNext()) {
final long row = cursor.next();
recordA.setRecordIndex(row);
if (filter.getBool(recordA)) {
rows.add(Rows.toRowID(partitionIndex, row));
found.addAt(index, symbolKey);
}
}
}
}
}
rows.sortAsUnsigned();
}
use of io.questdb.cairo.BitmapIndexReader in project questdb by bluestreak01.
the class LatestByValuesIndexedRecordCursor method buildTreeMap.
protected void buildTreeMap() {
final int keyCount = symbolKeys.size();
found.clear();
rows.setPos(0);
DataFrame frame;
while ((frame = this.dataFrameCursor.next()) != null && found.size() < keyCount) {
final BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex, BitmapIndexReader.DIR_BACKWARD);
final long rowLo = frame.getRowLo();
final long rowHi = frame.getRowHi() - 1;
for (int i = 0, n = symbolKeys.size(); i < n; i++) {
int symbolKey = symbolKeys.get(i);
int index = found.keyIndex(symbolKey);
if (index > -1) {
RowCursor cursor = indexReader.getCursor(false, symbolKey, rowLo, rowHi);
if (cursor.hasNext()) {
final long row = Rows.toRowID(frame.getPartitionIndex(), cursor.next());
rows.add(row);
found.addAt(index, symbolKey);
}
}
}
}
index = rows.size() - 1;
}
use of io.questdb.cairo.BitmapIndexReader in project questdb by bluestreak01.
the class LatestByValueIndexedFilteredRecordCursor method findRecord.
private void findRecord() {
DataFrame frame;
found = false;
while ((frame = this.dataFrameCursor.next()) != null) {
final int partitionIndex = frame.getPartitionIndex();
final BitmapIndexReader indexReader = frame.getBitmapIndexReader(columnIndex, BitmapIndexReader.DIR_BACKWARD);
final long rowLo = frame.getRowLo();
final long rowHi = frame.getRowHi() - 1;
this.recordA.jumpTo(partitionIndex, 0);
RowCursor cursor = indexReader.getCursor(false, symbolKey, rowLo, rowHi);
while (cursor.hasNext()) {
recordA.setRecordIndex(cursor.next());
if (filter.getBool(recordA)) {
found = true;
return;
}
}
}
}
Aggregations