use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class Mig506 method migrate.
static void migrate(MigrationContext migrationContext) {
// Update transaction file
// Before there was 1 int per symbol and list of removed partitions
// Now there is 2 ints per symbol and 4 longs per each non-removed partition
MigrationActions.LOG.info().$("rebuilding tx file [table=").$(migrationContext.getTablePath()).I$();
Path path = migrationContext.getTablePath();
final FilesFacade ff = migrationContext.getFf();
int pathDirLen = path.length();
path.concat(TXN_FILE_NAME).$();
if (!ff.exists(path)) {
MigrationActions.LOG.error().$("tx file does not exist, nothing to migrate [path=").$(path).I$();
return;
}
EngineMigration.backupFile(ff, path, migrationContext.getTablePath2(), TXN_FILE_NAME, 417);
MigrationActions.LOG.debug().$("opening for rw [path=").$(path).I$();
try (MemoryMARW txMem = migrationContext.createRwMemoryOf(ff, path.$())) {
long tempMem8b = migrationContext.getTempMemory(8);
MemoryARW txFileUpdate = migrationContext.getTempVirtualMem();
txFileUpdate.jumpTo(0);
int symbolColumnCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505);
for (int i = 0; i < symbolColumnCount; i++) {
final int symbolCount = txMem.getInt(MigrationActions.prefixedBlockOffset(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505, i + 1L, Integer.BYTES));
txFileUpdate.putInt(symbolCount);
txFileUpdate.putInt(symbolCount);
}
// Set partition segment size as 0 for now
long partitionSegmentOffset = txFileUpdate.getAppendOffset();
txFileUpdate.putInt(0);
final int partitionBy = TableUtils.readIntOrFail(ff, migrationContext.getMetadataFd(), TX_STRUCT_UPDATE_1_META_OFFSET_PARTITION_BY, tempMem8b, path);
if (partitionBy != PartitionBy.NONE) {
path.trimTo(pathDirLen);
writeAttachedPartitions(ff, tempMem8b, path, txMem, partitionBy, symbolColumnCount, txFileUpdate);
}
long updateSize = txFileUpdate.getAppendOffset();
long partitionSegmentSize = updateSize - partitionSegmentOffset - Integer.BYTES;
txFileUpdate.putInt(partitionSegmentOffset, (int) partitionSegmentSize);
// Save txFileUpdate to tx file starting at LOCAL_TX_OFFSET_MAP_WRITER_COUNT + 4
long writeOffset = MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + Integer.BYTES;
txMem.jumpTo(writeOffset);
for (int i = 0, size = 1; i < size && updateSize > 0; i++) {
long writeSize = Math.min(updateSize, txFileUpdate.getPageSize());
txMem.putBlockOfBytes(txFileUpdate.getPageAddress(i), writeSize);
updateSize -= writeSize;
}
assert updateSize == 0;
}
}
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class Mig600 method migrate.
static void migrate(MigrationContext migrationContext) {
MigrationActions.LOG.info().$("configuring default commit lag [table=").$(migrationContext.getTablePath()).I$();
final Path path = migrationContext.getTablePath();
final FilesFacade ff = migrationContext.getFf();
final long tempMem = migrationContext.getTempMemory(8);
final long fd = migrationContext.getMetadataFd();
TableUtils.writeIntOrFail(ff, fd, META_OFFSET_MAX_UNCOMMITTED_ROWS, migrationContext.getConfiguration().getMaxUncommittedRows(), tempMem, path);
TableUtils.writeLongOrFail(ff, fd, META_OFFSET_COMMIT_LAG, migrationContext.getConfiguration().getCommitLag(), tempMem, path);
}
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class Mig607 method migrate.
static void migrate(MigrationContext migrationContext) {
final FilesFacade ff = migrationContext.getFf();
Path path = migrationContext.getTablePath();
int plen = path.length();
path.trimTo(plen).concat(META_FILE_NAME).$();
long metaFileSize;
long txFileSize;
try (MemoryMARW metaMem = migrationContext.getRwMemory()) {
metaMem.of(ff, path, ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT);
final int columnCount = metaMem.getInt(0);
final int partitionBy = metaMem.getInt(4);
final long columnNameOffset = MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, columnCount, MigrationActions.META_COLUMN_DATA_SIZE_606);
try (MemoryMARW txMem = new MemoryCMARWImpl(ff, path.trimTo(plen).concat(TXN_FILE_NAME).$(), ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT)) {
// this is a variable length file; we need to count of symbol maps before we get to the partition
// table data
final int symbolMapCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505);
final long partitionCountOffset = MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 4 + symbolMapCount * 8L;
int partitionCount = txMem.getInt(partitionCountOffset) / Long.BYTES / LONGS_PER_TX_ATTACHED_PARTITION;
final long transientRowCount = txMem.getLong(TX_OFFSET_TRANSIENT_ROW_COUNT);
if (partitionBy != PartitionBy.NONE) {
for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
final long partitionDataOffset = partitionCountOffset + Integer.BYTES + partitionIndex * 8L * LONGS_PER_TX_ATTACHED_PARTITION;
setPathForPartition(path.trimTo(plen), partitionBy, txMem.getLong(partitionDataOffset), false);
// the row count may not be stored in _txn file for the last partition
// we need to use transient row count instead
long rowCount = partitionIndex < partitionCount - 1 ? txMem.getLong(partitionDataOffset + Long.BYTES) : transientRowCount;
long txSuffix = txMem.getLong(MigrationActions.prefixedBlockOffset(partitionDataOffset, 2, Long.BYTES));
if (txSuffix > -1) {
txnPartition(path, txSuffix);
}
migrate(ff, path, migrationContext, metaMem, columnCount, rowCount, columnNameOffset);
}
} else {
path.trimTo(plen).concat(DEFAULT_PARTITION_NAME);
migrate(ff, path, migrationContext, metaMem, columnCount, transientRowCount, columnNameOffset);
}
// update symbol maps
long tmpMem = migrationContext.getTempMemory();
int denseSymbolCount = 0;
long currentColumnNameOffset = columnNameOffset;
for (int i = 0; i < columnCount; i++) {
final CharSequence columnName = metaMem.getStr(currentColumnNameOffset);
currentColumnNameOffset += Vm.getStorageLength(columnName.length());
if (ColumnType.tagOf(metaMem.getInt(MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, i, MigrationActions.META_COLUMN_DATA_SIZE_606))) == ColumnType.SYMBOL) {
final int symbolCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 8 + denseSymbolCount * 8L);
final long offset = MigrationActions.prefixedBlockOffset(SymbolMapWriter.HEADER_SIZE, symbolCount, 8L);
SymbolMapWriter.offsetFileName(path.trimTo(plen), columnName);
long fd = TableUtils.openRW(ff, path, MigrationActions.LOG);
try {
long fileLen = ff.length(fd);
if (symbolCount > 0) {
if (fileLen < offset) {
MigrationActions.LOG.error().$("file is too short [path=").$(path).I$();
} else {
TableUtils.allocateDiskSpace(ff, fd, offset + 8);
long dataOffset = TableUtils.readLongOrFail(ff, fd, offset - 8L, tmpMem, path);
// string length
SymbolMapWriter.charFileName(path.trimTo(plen), columnName);
long fd2 = TableUtils.openRO(ff, path, MigrationActions.LOG);
try {
long len = TableUtils.readIntOrFail(ff, fd2, dataOffset, tmpMem, path);
if (len == -1) {
dataOffset += 4;
} else {
dataOffset += 4 + len * 2L;
}
TableUtils.writeLongOrFail(ff, fd, offset, dataOffset, tmpMem, path);
} finally {
ff.close(fd2);
}
}
}
} finally {
Vm.bestEffortClose(ff, MigrationActions.LOG, fd, true, offset + 8);
}
denseSymbolCount++;
}
}
txFileSize = txMem.getAppendOffset();
}
metaFileSize = metaMem.getAppendOffset();
}
// This migration when written originally used implementation of MemoryMARW which truncated files to size on close
// MemoryMARW now truncate to page size. To test old migrations here we simulate the migration as it is originally released
// So trim TX and META files to their sizes
path.trimTo(plen).concat(META_FILE_NAME).$();
trimFile(ff, path, metaFileSize);
path.trimTo(plen).concat(TXN_FILE_NAME).$();
trimFile(ff, path, txFileSize);
}
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class TableReader method reshuffleColumns.
private void reshuffleColumns(int columnCount, long pTransitionIndex) {
final long pIndexBase = pTransitionIndex + 8;
final long pState = pIndexBase + columnCount * 8L;
for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
int base = getColumnBase(partitionIndex);
try {
final Path path = pathGenPartitioned(partitionIndex).$();
final long partitionRowCount = openPartitionInfo.getQuick(partitionIndex * PARTITIONS_SLOT_SIZE + PARTITIONS_SLOT_OFFSET_SIZE);
Vect.memset(pState, columnCount, 0);
for (int i = 0; i < columnCount; i++) {
if (TableUtils.isEntryToBeProcessed(pState, i)) {
final int copyFrom = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L) - 1;
if (copyFrom == i) {
// It appears that column hasn't changed its position. There are three possibilities here:
// 1. Column has been deleted and re-added by the same name. We must check if file
// descriptor is still valid. If it isn't, reload the column from disk
// 2. Column has been forced out of the reader via closeColumnForRemove(). This is required
// on Windows before column can be deleted. In this case we must check for marker
// instance and the column from disk
// 3. Column hasn't been altered and we can skip to next column.
MemoryMR col = columns.getQuick(getPrimaryColumnIndex(base, i));
if ((col instanceof MemoryCMRImpl && col.isDeleted()) || col instanceof NullColumn) {
reloadColumnAt(path, columns, columnTops, bitmapIndexes, base, i, partitionRowCount);
}
continue;
}
if (copyFrom > -1) {
fetchColumnsFrom(base, copyFrom);
copyColumnsTo(this.columns, this.columnTops, this.bitmapIndexes, base, i, partitionRowCount);
int copyTo = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L + 4) - 1;
while (copyTo > -1 && TableUtils.isEntryToBeProcessed(pState, copyTo)) {
copyColumnsTo(this.columns, this.columnTops, this.bitmapIndexes, base, copyTo, partitionRowCount);
copyTo = Unsafe.getUnsafe().getInt(pIndexBase + (copyTo - 1) * 8L + 4);
}
Misc.free(tempCopyStruct.mem1);
Misc.free(tempCopyStruct.mem2);
Misc.free(tempCopyStruct.backwardReader);
Misc.free(tempCopyStruct.forwardReader);
} else {
// new instance
reloadColumnAt(path, columns, columnTops, bitmapIndexes, base, i, partitionRowCount);
}
}
}
for (int i = columnCount; i < this.columnCount; i++) {
int index = getPrimaryColumnIndex(base, i);
Misc.free(columns.getQuick(index));
Misc.free(columns.getQuick(index + 1));
}
} finally {
path.trimTo(rootLen);
}
}
}
use of io.questdb.std.str.Path in project questdb by bluestreak01.
the class TableReader method createBitmapIndexReaderAt.
private BitmapIndexReader createBitmapIndexReaderAt(int globalIndex, int columnBase, int columnIndex, int direction, long txn) {
BitmapIndexReader reader;
if (!metadata.isColumnIndexed(columnIndex)) {
throw CairoException.instance(0).put("Not indexed: ").put(metadata.getColumnName(columnIndex));
}
MemoryR col = columns.getQuick(globalIndex);
if (col instanceof NullColumn) {
if (direction == BitmapIndexReader.DIR_BACKWARD) {
reader = new BitmapIndexBwdNullReader();
bitmapIndexes.setQuick(globalIndex, reader);
} else {
reader = new BitmapIndexFwdNullReader();
bitmapIndexes.setQuick(globalIndex + 1, reader);
}
} else {
Path path = pathGenPartitioned(getPartitionIndex(columnBase));
try {
if (direction == BitmapIndexReader.DIR_BACKWARD) {
reader = new BitmapIndexBwdReader(configuration, path, metadata.getColumnName(columnIndex), getColumnTop(columnBase, columnIndex), txn);
bitmapIndexes.setQuick(globalIndex, reader);
} else {
reader = new BitmapIndexFwdReader(configuration, path, metadata.getColumnName(columnIndex), getColumnTop(columnBase, columnIndex), txn);
bitmapIndexes.setQuick(globalIndex + 1, reader);
}
} finally {
path.trimTo(rootLen);
}
}
return reader;
}
Aggregations