Examples with MemoryMR io.questdb.cairo.vm.api.MemoryMR used on opensource projects

Example 6 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TableReader method reloadPartition.

/**
 * Updates boundaries of all columns in partition.
 *
 * @param partitionIndex index of partition
 * @param rowCount       number of rows in partition
 */
private void reloadPartition(int partitionIndex, long rowCount, long openPartitionNameTxn) {
    Path path = pathGenPartitioned(partitionIndex);
    TableUtils.txnPartitionConditionally(path, openPartitionNameTxn);
    try {
        int symbolMapIndex = 0;
        int columnBase = getColumnBase(partitionIndex);
        for (int i = 0; i < columnCount; i++) {
            final int index = getPrimaryColumnIndex(columnBase, i);
            final MemoryMR mem1 = columns.getQuick(index);
            if (mem1 instanceof NullColumn) {
                reloadColumnAt(path, columns, columnTops, bitmapIndexes, columnBase, i, rowCount);
            } else {
                growColumn(mem1, columns.getQuick(index + 1), metadata.getColumnType(i), rowCount - getColumnTop(columnBase, i));
            }
            // reload symbol map
            SymbolMapReader reader = symbolMapReaders.getQuick(i);
            if (reader == null) {
                continue;
            }
            reader.updateSymbolCount(symbolCountSnapshot.getQuick(symbolMapIndex++));
        }
    } finally {
        path.trimTo(rootLen);
    }
}

Example 7 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TableReader method reloadColumnAt.

private void reloadColumnAt(Path path, ObjList<MemoryMR> columns, LongList columnTops, ObjList<BitmapIndexReader> indexReaders, int columnBase, int columnIndex, long partitionRowCount) {
    final int plen = path.length();
    try {
        final CharSequence name = metadata.getColumnName(columnIndex);
        final int primaryIndex = getPrimaryColumnIndex(columnBase, columnIndex);
        final int secondaryIndex = primaryIndex + 1;
        MemoryMR mem1 = columns.getQuick(primaryIndex);
        MemoryMR mem2 = columns.getQuick(secondaryIndex);
        final long columnTop = TableUtils.readColumnTop(ff, path.trimTo(plen), name, plen, tempMem8b, false);
        final long columnRowCount = partitionRowCount - columnTop;
        // column file. This makes it necessary to check for .d file existence
        if (partitionRowCount > 0 && ff.exists(TableUtils.dFile(path.trimTo(plen), name))) {
            final int columnType = metadata.getColumnType(columnIndex);
            if (ColumnType.isVariableLength(columnType)) {
                long columnSize = columnRowCount * 8L + 8L;
                TableUtils.iFile(path.trimTo(plen), name);
                mem2 = openOrCreateMemory(path, columns, secondaryIndex, mem2, columnSize);
                columnSize = mem2.getLong(columnRowCount * 8L);
                TableUtils.dFile(path.trimTo(plen), name);
                openOrCreateMemory(path, columns, primaryIndex, mem1, columnSize);
            } else {
                long columnSize = columnRowCount << ColumnType.pow2SizeOf(columnType);
                TableUtils.dFile(path.trimTo(plen), name);
                openOrCreateMemory(path, columns, primaryIndex, mem1, columnSize);
                Misc.free(columns.getAndSetQuick(secondaryIndex, null));
            }
            columnTops.setQuick(columnBase / 2 + columnIndex, columnTop);
            if (metadata.isColumnIndexed(columnIndex)) {
                BitmapIndexReader indexReader = indexReaders.getQuick(primaryIndex);
                if (indexReader instanceof BitmapIndexBwdReader) {
                    // name txn is -1 because the parent call sets up partition name for us
                    ((BitmapIndexBwdReader) indexReader).of(configuration, path.trimTo(plen), name, columnTop, -1);
                }
                indexReader = indexReaders.getQuick(secondaryIndex);
                if (indexReader instanceof BitmapIndexFwdReader) {
                    ((BitmapIndexFwdReader) indexReader).of(configuration, path.trimTo(plen), name, columnTop, -1);
                }
            } else {
                Misc.free(indexReaders.getAndSetQuick(primaryIndex, null));
                Misc.free(indexReaders.getAndSetQuick(secondaryIndex, null));
            }
        } else {
            Misc.free(columns.getAndSetQuick(primaryIndex, NullColumn.INSTANCE));
            Misc.free(columns.getAndSetQuick(secondaryIndex, NullColumn.INSTANCE));
            // the appropriate index for NUllColumn will be created lazily when requested
            // these indexes have state and may not be always required
            Misc.free(indexReaders.getAndSetQuick(primaryIndex, null));
            Misc.free(indexReaders.getAndSetQuick(secondaryIndex, null));
        }
    } finally {
        path.trimTo(plen);
    }
}

Example 8 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TableReader method createNewColumnList.

private void createNewColumnList(int columnCount, long pTransitionIndex, int columnBits) {
    int capacity = partitionCount << columnBits;
    final ObjList<MemoryMR> columns = new ObjList<>(capacity);
    final LongList columnTops = new LongList(capacity / 2);
    final ObjList<BitmapIndexReader> indexReaders = new ObjList<>(capacity);
    columns.setPos(capacity + 2);
    columns.setQuick(0, NullColumn.INSTANCE);
    columns.setQuick(1, NullColumn.INSTANCE);
    columnTops.setPos(capacity / 2);
    indexReaders.setPos(capacity + 2);
    final long pIndexBase = pTransitionIndex + 8;
    for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
        final int base = partitionIndex << columnBits;
        final int oldBase = partitionIndex << columnCountBits;
        try {
            final Path path = pathGenPartitioned(partitionIndex).$();
            long partitionRowCount = openPartitionInfo.getQuick(partitionIndex * PARTITIONS_SLOT_SIZE + PARTITIONS_SLOT_OFFSET_SIZE);
            for (int i = 0; i < columnCount; i++) {
                final int copyFrom = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L) - 1;
                if (copyFrom > -1) {
                    fetchColumnsFrom(oldBase, copyFrom);
                    copyColumnsTo(columns, columnTops, indexReaders, base, i, partitionRowCount);
                } else {
                    // new instance
                    reloadColumnAt(path, columns, columnTops, indexReaders, base, i, partitionRowCount);
                }
            }
            // free remaining columns
            for (int i = 0; i < this.columnCount; i++) {
                final int index = getPrimaryColumnIndex(oldBase, i);
                Misc.free(this.columns.getQuick(index));
                Misc.free(this.columns.getQuick(index + 1));
            }
        } finally {
            path.trimTo(rootLen);
        }
    }
    this.columns = columns;
    this.columnTops = columnTops;
    this.columnCountBits = columnBits;
    this.bitmapIndexes = indexReaders;
}

Example 9 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TableReader method copyColumnsTo.

private void copyColumnsTo(ObjList<MemoryMR> columns, LongList columnTops, ObjList<BitmapIndexReader> indexReaders, int columnBase, int columnIndex, long partitionRowCount) {
    MemoryMR mem1 = tempCopyStruct.mem1;
    final boolean reload = mem1 != null && mem1.isDeleted();
    final int index = getPrimaryColumnIndex(columnBase, columnIndex);
    tempCopyStruct.mem1 = columns.getAndSetQuick(index, mem1);
    tempCopyStruct.mem2 = columns.getAndSetQuick(index + 1, tempCopyStruct.mem2);
    tempCopyStruct.top = columnTops.getAndSetQuick(columnBase / 2 + columnIndex, tempCopyStruct.top);
    tempCopyStruct.backwardReader = indexReaders.getAndSetQuick(index, tempCopyStruct.backwardReader);
    tempCopyStruct.forwardReader = indexReaders.getAndSetQuick(index + 1, tempCopyStruct.forwardReader);
    if (reload) {
        reloadColumnAt(path, columns, columnTops, indexReaders, columnBase, columnIndex, partitionRowCount);
    }
}

Example 10 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TableReader method reshuffleColumns.

private void reshuffleColumns(int columnCount, long pTransitionIndex) {
    final long pIndexBase = pTransitionIndex + 8;
    final long pState = pIndexBase + columnCount * 8L;
    for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
        int base = getColumnBase(partitionIndex);
        try {
            final Path path = pathGenPartitioned(partitionIndex).$();
            final long partitionRowCount = openPartitionInfo.getQuick(partitionIndex * PARTITIONS_SLOT_SIZE + PARTITIONS_SLOT_OFFSET_SIZE);
            Vect.memset(pState, columnCount, 0);
            for (int i = 0; i < columnCount; i++) {
                if (TableUtils.isEntryToBeProcessed(pState, i)) {
                    final int copyFrom = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L) - 1;
                    if (copyFrom == i) {
                        // It appears that column hasn't changed its position. There are three possibilities here:
                        // 1. Column has been deleted and re-added by the same name. We must check if file
                        // descriptor is still valid. If it isn't, reload the column from disk
                        // 2. Column has been forced out of the reader via closeColumnForRemove(). This is required
                        // on Windows before column can be deleted. In this case we must check for marker
                        // instance and the column from disk
                        // 3. Column hasn't been altered and we can skip to next column.
                        MemoryMR col = columns.getQuick(getPrimaryColumnIndex(base, i));
                        if ((col instanceof MemoryCMRImpl && col.isDeleted()) || col instanceof NullColumn) {
                            reloadColumnAt(path, columns, columnTops, bitmapIndexes, base, i, partitionRowCount);
                        }
                        continue;
                    }
                    if (copyFrom > -1) {
                        fetchColumnsFrom(base, copyFrom);
                        copyColumnsTo(this.columns, this.columnTops, this.bitmapIndexes, base, i, partitionRowCount);
                        int copyTo = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L + 4) - 1;
                        while (copyTo > -1 && TableUtils.isEntryToBeProcessed(pState, copyTo)) {
                            copyColumnsTo(this.columns, this.columnTops, this.bitmapIndexes, base, copyTo, partitionRowCount);
                            copyTo = Unsafe.getUnsafe().getInt(pIndexBase + (copyTo - 1) * 8L + 4);
                        }
                        Misc.free(tempCopyStruct.mem1);
                        Misc.free(tempCopyStruct.mem2);
                        Misc.free(tempCopyStruct.backwardReader);
                        Misc.free(tempCopyStruct.forwardReader);
                    } else {
                        // new instance
                        reloadColumnAt(path, columns, columnTops, bitmapIndexes, base, i, partitionRowCount);
                    }
                }
            }
            for (int i = columnCount; i < this.columnCount; i++) {
                int index = getPrimaryColumnIndex(base, i);
                Misc.free(columns.getQuick(index));
                Misc.free(columns.getQuick(index + 1));
            }
        } finally {
            path.trimTo(rootLen);
        }
    }
}