Examples with Path io.questdb.std.str.Path used on opensource projects

Example 56 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class TableReader method reloadPartition.

/**
 * Updates boundaries of all columns in partition.
 *
 * @param partitionIndex index of partition
 * @param rowCount       number of rows in partition
 */
private void reloadPartition(int partitionIndex, long rowCount, long openPartitionNameTxn) {
    Path path = pathGenPartitioned(partitionIndex);
    TableUtils.txnPartitionConditionally(path, openPartitionNameTxn);
    try {
        int symbolMapIndex = 0;
        int columnBase = getColumnBase(partitionIndex);
        for (int i = 0; i < columnCount; i++) {
            final int index = getPrimaryColumnIndex(columnBase, i);
            final MemoryMR mem1 = columns.getQuick(index);
            if (mem1 instanceof NullColumn) {
                reloadColumnAt(path, columns, columnTops, bitmapIndexes, columnBase, i, rowCount);
            } else {
                growColumn(mem1, columns.getQuick(index + 1), metadata.getColumnType(i), rowCount - getColumnTop(columnBase, i));
            }
            // reload symbol map
            SymbolMapReader reader = symbolMapReaders.getQuick(i);
            if (reader == null) {
                continue;
            }
            reader.updateSymbolCount(symbolCountSnapshot.getQuick(symbolMapIndex++));
        }
    } finally {
        path.trimTo(rootLen);
    }
}

Example 57 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class TableReader method openPartition0.

private long openPartition0(int partitionIndex) {
    if (txFile.getPartitionCount() < 2 && txFile.getTransientRowCount() == 0) {
        return -1;
    }
    try {
        final long partitionNameTxn = txFile.getPartitionNameTxn(partitionIndex);
        Path path = pathGenPartitioned(partitionIndex);
        TableUtils.txnPartitionConditionally(path, partitionNameTxn);
        if (ff.exists(path.$())) {
            path.chop$();
            final long partitionSize = txFile.getPartitionSize(partitionIndex);
            LOG.info().$("open partition ").utf8(path.$()).$(" [rowCount=").$(partitionSize).$(", partitionNameTxn=").$(partitionNameTxn).$(", transientRowCount=").$(txFile.getTransientRowCount()).$(", partitionIndex=").$(partitionIndex).$(", partitionCount=").$(partitionCount).$(']').$();
            if (partitionSize > 0) {
                openPartitionColumns(path, getColumnBase(partitionIndex), partitionSize);
                final int offset = partitionIndex * PARTITIONS_SLOT_SIZE;
                this.openPartitionInfo.setQuick(offset + PARTITIONS_SLOT_OFFSET_SIZE, partitionSize);
            }
            return partitionSize;
        }
        LOG.error().$("open partition failed, partition does not exist on the disk. [path=").utf8(path.$()).I$();
        if (getPartitionedBy() != PartitionBy.NONE) {
            CairoException exception = CairoException.instance(0).put("Partition '");
            formatPartitionDirName(partitionIndex, exception.message);
            TableUtils.txnPartitionConditionally(exception.message, partitionNameTxn);
            exception.put("' does not exist in table '").put(tableName).put("' directory. Run [ALTER TABLE ").put(tableName).put(" DROP PARTITION LIST '");
            formatPartitionDirName(partitionIndex, exception.message);
            TableUtils.txnPartitionConditionally(exception.message, partitionNameTxn);
            exception.put("'] to repair the table or restore the partition directory.");
            throw exception;
        } else {
            throw CairoException.instance(0).put("Table '").put(tableName).put("' data directory does not exist on the disk at ").put(path).put(". Restore data on disk or drop the table.");
        }
    } finally {
        path.trimTo(rootLen);
    }
}

Example 58 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class TableReader method createNewColumnList.

private void createNewColumnList(int columnCount, long pTransitionIndex, int columnBits) {
    int capacity = partitionCount << columnBits;
    final ObjList<MemoryMR> columns = new ObjList<>(capacity);
    final LongList columnTops = new LongList(capacity / 2);
    final ObjList<BitmapIndexReader> indexReaders = new ObjList<>(capacity);
    columns.setPos(capacity + 2);
    columns.setQuick(0, NullColumn.INSTANCE);
    columns.setQuick(1, NullColumn.INSTANCE);
    columnTops.setPos(capacity / 2);
    indexReaders.setPos(capacity + 2);
    final long pIndexBase = pTransitionIndex + 8;
    for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) {
        final int base = partitionIndex << columnBits;
        final int oldBase = partitionIndex << columnCountBits;
        try {
            final Path path = pathGenPartitioned(partitionIndex).$();
            long partitionRowCount = openPartitionInfo.getQuick(partitionIndex * PARTITIONS_SLOT_SIZE + PARTITIONS_SLOT_OFFSET_SIZE);
            for (int i = 0; i < columnCount; i++) {
                final int copyFrom = Unsafe.getUnsafe().getInt(pIndexBase + i * 8L) - 1;
                if (copyFrom > -1) {
                    fetchColumnsFrom(oldBase, copyFrom);
                    copyColumnsTo(columns, columnTops, indexReaders, base, i, partitionRowCount);
                } else {
                    // new instance
                    reloadColumnAt(path, columns, columnTops, indexReaders, base, i, partitionRowCount);
                }
            }
            // free remaining columns
            for (int i = 0; i < this.columnCount; i++) {
                final int index = getPrimaryColumnIndex(oldBase, i);
                Misc.free(this.columns.getQuick(index));
                Misc.free(this.columns.getQuick(index + 1));
            }
        } finally {
            path.trimTo(rootLen);
        }
    }
    this.columns = columns;
    this.columnTops = columnTops;
    this.columnCountBits = columnBits;
    this.bitmapIndexes = indexReaders;
}

Example 59 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class EngineMigration method upgradeTables.

private static boolean upgradeTables(MigrationContext context, int latestVersion) {
    final FilesFacade ff = context.getFf();
    final CharSequence root = context.getConfiguration().getRoot();
    long mem = context.getTempMemory(8);
    final AtomicBoolean updateSuccess = new AtomicBoolean(true);
    try (Path path = new Path();
        Path copyPath = new Path()) {
        path.of(root);
        copyPath.of(root);
        final int rootLen = path.length();
        final NativeLPSZ nativeLPSZ = new NativeLPSZ();
        ff.iterateDir(path.$(), (name, type) -> {
            if (type == Files.DT_DIR) {
                nativeLPSZ.of(name);
                if (Chars.notDots(nativeLPSZ)) {
                    path.trimTo(rootLen);
                    path.concat(nativeLPSZ);
                    copyPath.trimTo(rootLen);
                    copyPath.concat(nativeLPSZ);
                    final int plen = path.length();
                    path.concat(TableUtils.META_FILE_NAME);
                    if (ff.exists(path.$())) {
                        final long fd = openFileRWOrFail(ff, path);
                        try {
                            int currentTableVersion = TableUtils.readIntOrFail(ff, fd, META_OFFSET_VERSION, mem, path);
                            if (currentTableVersion < latestVersion) {
                                LOG.info().$("upgrading [path=").$(path).$(",fromVersion=").$(currentTableVersion).$(",toVersion=").$(latestVersion).I$();
                                copyPath.trimTo(plen);
                                backupFile(ff, path, copyPath, TableUtils.META_FILE_NAME, currentTableVersion);
                                path.trimTo(plen);
                                context.of(path, copyPath, fd);
                                for (int ver = currentTableVersion + 1; ver <= latestVersion; ver++) {
                                    final MigrationAction migration = getMigrationToVersion(ver);
                                    if (migration != null) {
                                        try {
                                            LOG.info().$("upgrading table [path=").$(path).$(",toVersion=").$(ver).I$();
                                            migration.migrate(context);
                                            path.trimTo(plen);
                                        } catch (Throwable e) {
                                            LOG.error().$("failed to upgrade table path=").$(path.trimTo(plen)).$(", exception: ").$(e).$();
                                            throw e;
                                        }
                                    }
                                    TableUtils.writeIntOrFail(ff, fd, META_OFFSET_VERSION, ver, mem, path.trimTo(plen));
                                }
                            }
                        } finally {
                            ff.close(fd);
                            path.trimTo(plen);
                            copyPath.trimTo(plen);
                        }
                    }
                }
            }
        });
        LOG.info().$("upgraded tables to ").$(latestVersion).$();
    }
    return updateSuccess.get();
}

Example 60 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class TextLoaderTest method testReservedTableName.

@Test
public void testReservedTableName() throws Exception {
    assertNoLeak(textLoader -> {
        String csv = "\"\"\"CMP2\",8,8000,2.27636352181435,2015-01-29T19:15:09.000Z,2015-01-29 19:15:09,01/29/2015,323,TRUE,14925407\n" + "CMP1,2,1581,9.01423481060192,2015-01-30T19:15:09.000Z,2015-01-30 19:15:09,01/30/2015,9138,FALSE,68225213\n" + "CMP2,8,7067,9.6284336107783,2015-01-31T19:15:09.000Z,2015-01-31 19:15:09,01/31/2015,8197,TRUE,58403960\n" + "CMP1,8,5313,8.87764661805704,2015-02-01T19:15:09.000Z,2015-02-01 19:15:09,02/01/2015,2733,FALSE,69698373\n" + ",4,3883,7.96873019309714,2015-02-02T19:15:09.000Z,2015-02-02 19:15:09,02/02/2015,6912,TRUE,91147394\n" + "CMP1,7,4256,2.46553522534668,2015-02-03T19:15:09.000Z,2015-02-03 19:15:09,02/03/2015,9453,FALSE,50278940\n" + "CMP2,4,155,5.08547495584935,2015-02-04T19:15:09.000Z,2015-02-04 19:15:09,02/04/2015,8919,TRUE,8671995\n" + "\"CMP1\",7,4486,,2015-02-05T19:15:09.000Z,2015-02-05 19:15:09,02/05/2015,8670,FALSE,751877\n" + "CMP2,2,6641,0.0381825352087617,2015-02-06T19:15:09.000Z,2015-02-06 19:15:09,02/06/2015,8331,TRUE,40909232527\n" + "CMP1,1,3579,0.849663221742958,2015-02-07T19:15:09.000Z,2015-02-07 19:15:09,02/07/2015,9592,FALSE,11490662\n" + "CMP2,2,4770,2.85092033445835,2015-02-08T19:15:09.000Z,2015-02-08 19:15:09,02/08/2015,253,TRUE,33766814\n" + "CMP1,5,4938,4.42754498450086,2015-02-09T19:15:09.000Z,2015-02-09 19:15:09,02/09/2015,7817,FALSE,61983099\n";
        try (Path path = new Path()) {
            path.of(configuration.getRoot()).concat("test").$();
            Files.touch(path);
        }
        configureLoaderDefaults(textLoader, (byte) -1, Atomicity.SKIP_ROW, true);
        try {
            playText0(textLoader, csv, 1024, ENTITY_MANIPULATOR);
            Assert.fail();
        } catch (CairoException e) {
            TestUtils.assertContains(e.getFlyweightMessage(), "name is reserved");
        }
    });
}