Search in sources :

Example 16 with FilesFacade

use of io.questdb.std.FilesFacade in project questdb by bluestreak01.

the class TxnScoreboardTest method testCleanOnExclusiveOpenLocksFile.

@Test
public void testCleanOnExclusiveOpenLocksFile() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (final Path shmPath = new Path()) {
            FilesFacade ff = FilesFacadeImpl.INSTANCE;
            try (final TxnScoreboard scoreboard = new TxnScoreboard(ff, shmPath.of(root), 1024)) {
                for (int i = 0; i < 1500; i++) {
                    scoreboard.acquireTxn(i);
                    scoreboard.releaseTxn(i);
                }
                Assert.assertEquals(1499, scoreboard.getMin());
            }
            // second open is exclusive, file should be truncated
            try (final TxnScoreboard scoreboard2 = new TxnScoreboard(ff, shmPath.of(root), 2048)) {
                Assert.assertEquals(0, scoreboard2.getMin());
                for (int i = 0; i < 10; i++) {
                    scoreboard2.acquireTxn(i);
                    scoreboard2.releaseTxn(i);
                }
                // This should not obtain exclusive lock even though file was empty when scoreboard2 put shared lock
                try (final TxnScoreboard scoreboard3 = new TxnScoreboard(ff, shmPath.of(root), 2048)) {
                    Assert.assertEquals(9, scoreboard2.getMin());
                    Assert.assertEquals(9, scoreboard3.getMin());
                }
            }
        }
    });
}
Also used : Path(io.questdb.std.str.Path) FilesFacade(io.questdb.std.FilesFacade) Test(org.junit.Test)

Example 17 with FilesFacade

use of io.questdb.std.FilesFacade in project questdb by bluestreak01.

the class Mig506 method migrate.

static void migrate(MigrationContext migrationContext) {
    // Update transaction file
    // Before there was 1 int per symbol and list of removed partitions
    // Now there is 2 ints per symbol and 4 longs per each non-removed partition
    MigrationActions.LOG.info().$("rebuilding tx file [table=").$(migrationContext.getTablePath()).I$();
    Path path = migrationContext.getTablePath();
    final FilesFacade ff = migrationContext.getFf();
    int pathDirLen = path.length();
    path.concat(TXN_FILE_NAME).$();
    if (!ff.exists(path)) {
        MigrationActions.LOG.error().$("tx file does not exist, nothing to migrate [path=").$(path).I$();
        return;
    }
    EngineMigration.backupFile(ff, path, migrationContext.getTablePath2(), TXN_FILE_NAME, 417);
    MigrationActions.LOG.debug().$("opening for rw [path=").$(path).I$();
    try (MemoryMARW txMem = migrationContext.createRwMemoryOf(ff, path.$())) {
        long tempMem8b = migrationContext.getTempMemory(8);
        MemoryARW txFileUpdate = migrationContext.getTempVirtualMem();
        txFileUpdate.jumpTo(0);
        int symbolColumnCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505);
        for (int i = 0; i < symbolColumnCount; i++) {
            final int symbolCount = txMem.getInt(MigrationActions.prefixedBlockOffset(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505, i + 1L, Integer.BYTES));
            txFileUpdate.putInt(symbolCount);
            txFileUpdate.putInt(symbolCount);
        }
        // Set partition segment size as 0 for now
        long partitionSegmentOffset = txFileUpdate.getAppendOffset();
        txFileUpdate.putInt(0);
        final int partitionBy = TableUtils.readIntOrFail(ff, migrationContext.getMetadataFd(), TX_STRUCT_UPDATE_1_META_OFFSET_PARTITION_BY, tempMem8b, path);
        if (partitionBy != PartitionBy.NONE) {
            path.trimTo(pathDirLen);
            writeAttachedPartitions(ff, tempMem8b, path, txMem, partitionBy, symbolColumnCount, txFileUpdate);
        }
        long updateSize = txFileUpdate.getAppendOffset();
        long partitionSegmentSize = updateSize - partitionSegmentOffset - Integer.BYTES;
        txFileUpdate.putInt(partitionSegmentOffset, (int) partitionSegmentSize);
        // Save txFileUpdate to tx file starting at LOCAL_TX_OFFSET_MAP_WRITER_COUNT + 4
        long writeOffset = MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + Integer.BYTES;
        txMem.jumpTo(writeOffset);
        for (int i = 0, size = 1; i < size && updateSize > 0; i++) {
            long writeSize = Math.min(updateSize, txFileUpdate.getPageSize());
            txMem.putBlockOfBytes(txFileUpdate.getPageAddress(i), writeSize);
            updateSize -= writeSize;
        }
        assert updateSize == 0;
    }
}
Also used : Path(io.questdb.std.str.Path) FilesFacade(io.questdb.std.FilesFacade) MemoryMARW(io.questdb.cairo.vm.api.MemoryMARW) MemoryARW(io.questdb.cairo.vm.api.MemoryARW)

Example 18 with FilesFacade

use of io.questdb.std.FilesFacade in project questdb by bluestreak01.

the class Mig600 method migrate.

static void migrate(MigrationContext migrationContext) {
    MigrationActions.LOG.info().$("configuring default commit lag [table=").$(migrationContext.getTablePath()).I$();
    final Path path = migrationContext.getTablePath();
    final FilesFacade ff = migrationContext.getFf();
    final long tempMem = migrationContext.getTempMemory(8);
    final long fd = migrationContext.getMetadataFd();
    TableUtils.writeIntOrFail(ff, fd, META_OFFSET_MAX_UNCOMMITTED_ROWS, migrationContext.getConfiguration().getMaxUncommittedRows(), tempMem, path);
    TableUtils.writeLongOrFail(ff, fd, META_OFFSET_COMMIT_LAG, migrationContext.getConfiguration().getCommitLag(), tempMem, path);
}
Also used : Path(io.questdb.std.str.Path) FilesFacade(io.questdb.std.FilesFacade)

Example 19 with FilesFacade

use of io.questdb.std.FilesFacade in project questdb by bluestreak01.

the class Mig607 method migrate.

static void migrate(MigrationContext migrationContext) {
    final FilesFacade ff = migrationContext.getFf();
    Path path = migrationContext.getTablePath();
    int plen = path.length();
    path.trimTo(plen).concat(META_FILE_NAME).$();
    long metaFileSize;
    long txFileSize;
    try (MemoryMARW metaMem = migrationContext.getRwMemory()) {
        metaMem.of(ff, path, ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT);
        final int columnCount = metaMem.getInt(0);
        final int partitionBy = metaMem.getInt(4);
        final long columnNameOffset = MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, columnCount, MigrationActions.META_COLUMN_DATA_SIZE_606);
        try (MemoryMARW txMem = new MemoryCMARWImpl(ff, path.trimTo(plen).concat(TXN_FILE_NAME).$(), ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT)) {
            // this is a variable length file; we need to count of symbol maps before we get to the partition
            // table data
            final int symbolMapCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505);
            final long partitionCountOffset = MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 4 + symbolMapCount * 8L;
            int partitionCount = txMem.getInt(partitionCountOffset) / Long.BYTES / LONGS_PER_TX_ATTACHED_PARTITION;
            final long transientRowCount = txMem.getLong(TX_OFFSET_TRANSIENT_ROW_COUNT);
            if (partitionBy != PartitionBy.NONE) {
                for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
                    final long partitionDataOffset = partitionCountOffset + Integer.BYTES + partitionIndex * 8L * LONGS_PER_TX_ATTACHED_PARTITION;
                    setPathForPartition(path.trimTo(plen), partitionBy, txMem.getLong(partitionDataOffset), false);
                    // the row count may not be stored in _txn file for the last partition
                    // we need to use transient row count instead
                    long rowCount = partitionIndex < partitionCount - 1 ? txMem.getLong(partitionDataOffset + Long.BYTES) : transientRowCount;
                    long txSuffix = txMem.getLong(MigrationActions.prefixedBlockOffset(partitionDataOffset, 2, Long.BYTES));
                    if (txSuffix > -1) {
                        txnPartition(path, txSuffix);
                    }
                    migrate(ff, path, migrationContext, metaMem, columnCount, rowCount, columnNameOffset);
                }
            } else {
                path.trimTo(plen).concat(DEFAULT_PARTITION_NAME);
                migrate(ff, path, migrationContext, metaMem, columnCount, transientRowCount, columnNameOffset);
            }
            // update symbol maps
            long tmpMem = migrationContext.getTempMemory();
            int denseSymbolCount = 0;
            long currentColumnNameOffset = columnNameOffset;
            for (int i = 0; i < columnCount; i++) {
                final CharSequence columnName = metaMem.getStr(currentColumnNameOffset);
                currentColumnNameOffset += Vm.getStorageLength(columnName.length());
                if (ColumnType.tagOf(metaMem.getInt(MigrationActions.prefixedBlockOffset(MigrationActions.META_OFFSET_COLUMN_TYPES_606, i, MigrationActions.META_COLUMN_DATA_SIZE_606))) == ColumnType.SYMBOL) {
                    final int symbolCount = txMem.getInt(MigrationActions.TX_OFFSET_MAP_WRITER_COUNT_505 + 8 + denseSymbolCount * 8L);
                    final long offset = MigrationActions.prefixedBlockOffset(SymbolMapWriter.HEADER_SIZE, symbolCount, 8L);
                    SymbolMapWriter.offsetFileName(path.trimTo(plen), columnName);
                    long fd = TableUtils.openRW(ff, path, MigrationActions.LOG);
                    try {
                        long fileLen = ff.length(fd);
                        if (symbolCount > 0) {
                            if (fileLen < offset) {
                                MigrationActions.LOG.error().$("file is too short [path=").$(path).I$();
                            } else {
                                TableUtils.allocateDiskSpace(ff, fd, offset + 8);
                                long dataOffset = TableUtils.readLongOrFail(ff, fd, offset - 8L, tmpMem, path);
                                // string length
                                SymbolMapWriter.charFileName(path.trimTo(plen), columnName);
                                long fd2 = TableUtils.openRO(ff, path, MigrationActions.LOG);
                                try {
                                    long len = TableUtils.readIntOrFail(ff, fd2, dataOffset, tmpMem, path);
                                    if (len == -1) {
                                        dataOffset += 4;
                                    } else {
                                        dataOffset += 4 + len * 2L;
                                    }
                                    TableUtils.writeLongOrFail(ff, fd, offset, dataOffset, tmpMem, path);
                                } finally {
                                    ff.close(fd2);
                                }
                            }
                        }
                    } finally {
                        Vm.bestEffortClose(ff, MigrationActions.LOG, fd, true, offset + 8);
                    }
                    denseSymbolCount++;
                }
            }
            txFileSize = txMem.getAppendOffset();
        }
        metaFileSize = metaMem.getAppendOffset();
    }
    // This migration when written originally used implementation of MemoryMARW which truncated files to size on close
    // MemoryMARW now truncate to page size. To test old migrations here we simulate the migration as it is originally released
    // So trim TX and META files to their sizes
    path.trimTo(plen).concat(META_FILE_NAME).$();
    trimFile(ff, path, metaFileSize);
    path.trimTo(plen).concat(TXN_FILE_NAME).$();
    trimFile(ff, path, txFileSize);
}
Also used : Path(io.questdb.std.str.Path) FilesFacade(io.questdb.std.FilesFacade) MemoryMARW(io.questdb.cairo.vm.api.MemoryMARW) MemoryCMARWImpl(io.questdb.cairo.vm.MemoryCMARWImpl)

Example 20 with FilesFacade

use of io.questdb.std.FilesFacade in project questdb by bluestreak01.

the class Mig605 method migrate.

static void migrate(MigrationContext migrationContext) {
    MigrationActions.LOG.info().$("updating column type IDs [table=").$(migrationContext.getTablePath()).I$();
    final FilesFacade ff = migrationContext.getFf();
    Path path = migrationContext.getTablePath();
    path.concat(META_FILE_NAME).$();
    if (!ff.exists(path)) {
        MigrationActions.LOG.error().$("meta file does not exist, nothing to migrate [path=").$(path).I$();
        return;
    }
    // Metadata file should already be backed up
    try (final MemoryMARW rwMem = migrationContext.getRwMemory()) {
        rwMem.of(ff, path, ff.getPageSize(), ff.length(path), MemoryTag.NATIVE_DEFAULT);
        // column count
        final int columnCount = rwMem.getInt(TableUtils.META_OFFSET_COUNT);
        long offset = TableUtils.META_OFFSET_COLUMN_TYPES;
        for (int i = 0; i < columnCount; i++) {
            final byte oldTypeId = rwMem.getByte(offset);
            final long oldFlags = rwMem.getLong(offset + 1);
            final int blockCapacity = rwMem.getInt(offset + 1 + 8);
            // column type id is int now
            // we grabbed 3 reserved bytes for extra type info
            // extra for old types is zeros
            // ColumnType.VERSION_420 - ColumnType.VERSION_419 = 1 except for BINARY, old 13 new 18
            rwMem.putInt(offset, oldTypeId == 13 ? 18 : oldTypeId + 1);
            rwMem.putLong(offset + 4, oldFlags);
            rwMem.putInt(offset + 4 + 8, blockCapacity);
            // old TableUtils.META_COLUMN_DATA_SIZE;
            offset += 16;
        }
    }
}
Also used : Path(io.questdb.std.str.Path) FilesFacade(io.questdb.std.FilesFacade) MemoryMARW(io.questdb.cairo.vm.api.MemoryMARW)

Aggregations

FilesFacade (io.questdb.std.FilesFacade)29 FilesFacadeImpl (io.questdb.std.FilesFacadeImpl)14 Test (org.junit.Test)14 Path (io.questdb.std.str.Path)13 MemoryMARW (io.questdb.cairo.vm.api.MemoryMARW)6 AbstractGriffinTest (io.questdb.griffin.AbstractGriffinTest)4 RecordCursorFactory (io.questdb.cairo.sql.RecordCursorFactory)3 SqlCompiler (io.questdb.griffin.SqlCompiler)3 LPSZ (io.questdb.std.str.LPSZ)3 MemoryCMARWImpl (io.questdb.cairo.vm.MemoryCMARWImpl)2 Rnd (io.questdb.std.Rnd)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 TableModel (io.questdb.cairo.TableModel)1 MemoryARW (io.questdb.cairo.vm.api.MemoryARW)1 LineTcpSender (io.questdb.cutlass.line.LineTcpSender)1 SqlException (io.questdb.griffin.SqlException)1 BindVariableServiceImpl (io.questdb.griffin.engine.functions.bind.BindVariableServiceImpl)1 DateFormat (io.questdb.std.datetime.DateFormat)1 TimestampFormatCompiler (io.questdb.std.datetime.microtime.TimestampFormatCompiler)1 File (java.io.File)1