Examples with MemoryCMARWImpl io.questdb.cairo.vm.MemoryCMARWImpl used on opensource projects

Example 1 with MemoryCMARWImpl

use of io.questdb.cairo.vm.MemoryCMARWImpl in project questdb by bluestreak01.

the class Mig607 method migrate.

static void migrate(MigrationContext migrationContext) {
    final FilesFacade ff = migrationContext.getFf();
    Path path = migrationContext.getTablePath();
    int plen = path.length();
    path.trimTo(plen).concat(META_FILE_NAME).$();
    long metaFileSize;
    long txFileSize;
    try (MemoryMARW metaMem = migrationContext.getRwMemory()) {
        metaMem.of(ff, path, ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT);
        final int columnCount = metaMem.getInt(0);
        final int partitionBy = metaMem.getInt(4);
        final long columnNameOffset = MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, columnCount, MigrationActions.META_COLUMN_DATA_SIZE_606);
        try (MemoryMARW txMem = new MemoryCMARWImpl(ff, path.trimTo(plen).concat(TXN_FILE_NAME).$(), ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT)) {
            // this is a variable length file; we need to count of symbol maps before we get to the partition
            // table data
            final int symbolMapCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505);
            final long partitionCountOffset = MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 4 + symbolMapCount * 8L;
            int partitionCount = txMem.getInt(partitionCountOffset) / Long.BYTES / LONGS_PER_TX_ATTACHED_PARTITION;
            final long transientRowCount = txMem.getLong(TX_OFFSET_TRANSIENT_ROW_COUNT);
            if (partitionBy != PartitionBy.NONE) {
                for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
                    final long partitionDataOffset = partitionCountOffset + Integer.BYTES + partitionIndex * 8L * LONGS_PER_TX_ATTACHED_PARTITION;
                    setPathForPartition(path.trimTo(plen), partitionBy, txMem.getLong(partitionDataOffset), false);
                    // the row count may not be stored in _txn file for the last partition
                    // we need to use transient row count instead
                    long rowCount = partitionIndex < partitionCount - 1 ? txMem.getLong(partitionDataOffset + Long.BYTES) : transientRowCount;
                    long txSuffix = txMem.getLong(MigrationActions.prefixedBlockOffset(partitionDataOffset, 2, Long.BYTES));
                    if (txSuffix > -1) {
                        txnPartition(path, txSuffix);
                    }
                    migrate(ff, path, migrationContext, metaMem, columnCount, rowCount, columnNameOffset);
                }
            } else {
                path.trimTo(plen).concat(DEFAULT_PARTITION_NAME);
                migrate(ff, path, migrationContext, metaMem, columnCount, transientRowCount, columnNameOffset);
            }
            // update symbol maps
            long tmpMem = migrationContext.getTempMemory();
            int denseSymbolCount = 0;
            long currentColumnNameOffset = columnNameOffset;
            for (int i = 0; i < columnCount; i++) {
                final CharSequence columnName = metaMem.getStr(currentColumnNameOffset);
                currentColumnNameOffset += Vm.getStorageLength(columnName.length());
                if (ColumnType.tagOf(metaMem.getInt(MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, i, MigrationActions.META_COLUMN_DATA_SIZE_606))) == ColumnType.SYMBOL) {
                    final int symbolCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 8 + denseSymbolCount * 8L);
                    final long offset = MigrationActions.prefixedBlockOffset(SymbolMapWriter.HEADER_SIZE, symbolCount, 8L);
                    SymbolMapWriter.offsetFileName(path.trimTo(plen), columnName);
                    long fd = TableUtils.openRW(ff, path, MigrationActions.LOG);
                    try {
                        long fileLen = ff.length(fd);
                        if (symbolCount > 0) {
                            if (fileLen < offset) {
                                MigrationActions.LOG.error().$("file is too short [path=").$(path).I$();
                            } else {
                                TableUtils.allocateDiskSpace(ff, fd, offset + 8);
                                long dataOffset = TableUtils.readLongOrFail(ff, fd, offset - 8L, tmpMem, path);
                                // string length
                                SymbolMapWriter.charFileName(path.trimTo(plen), columnName);
                                long fd2 = TableUtils.openRO(ff, path, MigrationActions.LOG);
                                try {
                                    long len = TableUtils.readIntOrFail(ff, fd2, dataOffset, tmpMem, path);
                                    if (len == -1) {
                                        dataOffset += 4;
                                    } else {
                                        dataOffset += 4 + len * 2L;
                                    }
                                    TableUtils.writeLongOrFail(ff, fd, offset, dataOffset, tmpMem, path);
                                } finally {
                                    ff.close(fd2);
                                }
                            }
                        }
                    } finally {
                        Vm.bestEffortClose(ff, MigrationActions.LOG, fd, true, offset + 8);
                    }
                    denseSymbolCount++;
                }
            }
            txFileSize = txMem.getAppendOffset();
        }
        metaFileSize = metaMem.getAppendOffset();
    }
    // This migration when written originally used implementation of MemoryMARW which truncated files to size on close
    // MemoryMARW now truncate to page size. To test old migrations here we simulate the migration as it is originally released
    // So trim TX and META files to their sizes
    path.trimTo(plen).concat(META_FILE_NAME).$();
    trimFile(ff, path, metaFileSize);
    path.trimTo(plen).concat(TXN_FILE_NAME).$();
    trimFile(ff, path, txFileSize);
}

Example 2 with MemoryCMARWImpl

use of io.questdb.cairo.vm.MemoryCMARWImpl in project questdb by bluestreak01.

the class Mig609 method migrate.

static void migrate(MigrationContext migrationContext) {
    final FilesFacade ff = migrationContext.getFf();
    final Path path = migrationContext.getTablePath();
    final int plen = path.length();
    path.trimTo(plen).concat(META_FILE_NAME).$();
    try (MemoryMARW metaMem = migrationContext.getRwMemory()) {
        metaMem.of(ff, path, ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT);
        // we require partition by value to avoid processing non-partitioned tables
        final int partitionBy = metaMem.getInt(4);
        try (MemoryMARW txMem = new MemoryCMARWImpl(ff, path.trimTo(plen).concat(TXN_FILE_NAME).$(), ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT)) {
            // this is a variable length file; we need to count of symbol maps before we get to the partition
            // table data
            final int symbolMapCount = txMem.getInt(TX_OFFSET_MAP_WRITER_COUNT_608);
            final long partitionCountOffset = TX_OFFSET_MAP_WRITER_COUNT_608 + 4 + symbolMapCount * 8L;
            // walk only non-active partition to extract sizes
            final int partitionCount = txMem.getInt(partitionCountOffset) / Long.BYTES / LONGS_PER_TX_ATTACHED_PARTITION - 1;
            if (partitionBy != PartitionBy.NONE) {
                long calculatedFixedRowCount = 0;
                for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
                    final long partitionDataOffset = partitionCountOffset + Integer.BYTES + partitionIndex * 8L * LONGS_PER_TX_ATTACHED_PARTITION;
                    // the row count may not be stored in _txn file for the last partition
                    // we need to use transient row count instead
                    calculatedFixedRowCount += txMem.getLong(partitionDataOffset + Long.BYTES);
                }
                long currentFixedRowCount = txMem.getLong(TX_OFFSET_FIXED_ROW_COUNT_505);
                if (currentFixedRowCount != calculatedFixedRowCount) {
                    txMem.putLong(TX_OFFSET_FIXED_ROW_COUNT_505, calculatedFixedRowCount);
                    LOG.info().$("fixed row count is out [table=").$(path.trimTo(plen).$()).$(", currentFixedRowCount=").$(currentFixedRowCount).$(", calculatedFixedRowCount=").$(calculatedFixedRowCount).I$();
                }
            }
        }
    }
}