Examples with NativeLPSZ com.questdb.std.str.NativeLPSZ used on opensource projects

Example 1 with NativeLPSZ

use of com.questdb.std.str.NativeLPSZ in project questdb by bluestreak01.

the class $ColsRecordSource method prepareCursor.

@Override
public RecordCursor prepareCursor(ReaderFactory factory, CancellationHandler cancellationHandler) {
    records.clear();
    NativeLPSZ name = tlNativeLpsz.get();
    int bufSz = metaSize;
    long buf = Unsafe.malloc(bufSz);
    DirectCharSequence dcs = tlDcs.get();
    try {
        String base = factory.getConfiguration().getJournalBase().getAbsolutePath();
        Path path = tlPath.get().of(base).$();
        long find = Files.findFirst(path);
        if (find > 0) {
            try {
                long p = -1;
                Path compositePath = tlCompositePath.get();
                do {
                    cancellationHandler.check();
                    if (Files.findType(find) == Files.DT_DIR) {
                        name.of(Files.findName(find));
                        if (Files.isDots(name)) {
                            continue;
                        }
                        compositePath.of(base).concat(name).concat(JournalConfiguration.FILE_NAME).$();
                        if (Files.exists(compositePath)) {
                            long fd = Files.openRO(compositePath);
                            if (fd < 0) {
                                LOG.error().$("Cannot open: ").$(compositePath).$(" [").$(Os.errno()).$(']').$();
                                continue;
                            }
                            int columnCount;
                            int partitionBy;
                            int timestampIndex;
                            try {
                                long len = Files.length(compositePath);
                                // don't bother with huge meta files. There is a high chance of them being fake.
                                if (len > maxMetaSize) {
                                    LOG.error().$("File : ").$(compositePath).$(" is too large [").$(len).$(']').$();
                                    continue;
                                }
                                if (len > bufSz) {
                                    Unsafe.free(buf, bufSz);
                                    buf = Unsafe.malloc(bufSz = (int) len);
                                }
                                Files.read(fd, buf, (int) len, 0);
                                // skip over append offset
                                long readPtr = buf + 8;
                                // skip over ID string
                                readPtr += Unsafe.getUnsafe().getInt(readPtr) * 2 + 4;
                                // // skip over location string
                                // readPtr += Unsafe.getUnsafe().getInt(readPtr) * 2 + 4;
                                partitionBy = Unsafe.getUnsafe().getInt(readPtr);
                                columnCount = Unsafe.getUnsafe().getInt(readPtr + 4);
                                timestampIndex = Unsafe.getUnsafe().getInt(readPtr + 8);
                                readPtr += 12;
                                for (int i = 0; i < columnCount; i++) {
                                    p = records.beginRecord(p);
                                    // table_name
                                    records.appendStr(name);
                                    // column_name
                                    int l = Unsafe.getUnsafe().getInt(readPtr) * 2;
                                    readPtr += 4;
                                    records.appendStr(dcs.of(readPtr, readPtr + l));
                                    readPtr += l;
                                    // column_type
                                    records.appendInt(Unsafe.getUnsafe().getInt(readPtr));
                                    // skip size and avgsize
                                    readPtr += 12;
                                    // timestamp
                                    records.appendBool(i == timestampIndex);
                                    // partition_by
                                    records.appendInt(i == timestampIndex ? partitionBy : SymbolTable.VALUE_IS_NULL);
                                    // indexed
                                    records.appendBool(Unsafe.getBool(readPtr));
                                    // skip 2 ints
                                    readPtr += 9;
                                    // index_buckets
                                    records.appendInt(Unsafe.getUnsafe().getInt(readPtr));
                                    readPtr += 4;
                                    l = Unsafe.getUnsafe().getInt(readPtr);
                                    readPtr += 4;
                                    if (l > -1) {
                                        readPtr += (l * 2);
                                    }
                                    readPtr += 1;
                                }
                            } finally {
                                Files.close(fd);
                            }
                        }
                    }
                } while (Files.findNext(find) > 0);
            } finally {
                Files.findClose(find);
            }
        }
    } finally {
        Unsafe.free(buf, bufSz);
    }
    records.toTop();
    return records;
}

Example 2 with NativeLPSZ

use of com.questdb.std.str.NativeLPSZ in project questdb by bluestreak01.

the class $TabsRecordSource method sumSizes.

private static long sumSizes(LPSZ path) {
    long total = 0;
    long find = Files.findFirst(path);
    if (find == 0) {
        return 0;
    }
    try (Path cp = new Path()) {
        NativeLPSZ file = tlNativeLpsz.get();
        try {
            do {
                file.of(Files.findName(find));
                if (Files.isDots(file)) {
                    continue;
                }
                LPSZ n = cp.of(path).concat(file).$();
                if (Files.findType(find) == Files.DT_DIR) {
                    total += sumSizes(n);
                } else {
                    total += Files.length(n);
                }
            } while (Files.findNext(find) > 0);
        } finally {
            Files.findClose(find);
        }
    }
    return total;
}

Example 3 with NativeLPSZ

use of com.questdb.std.str.NativeLPSZ in project questdb by bluestreak01.

the class $TabsRecordSource method prepareCursor.

@Override
public RecordCursor prepareCursor(ReaderFactory factory, CancellationHandler cancellationHandler) {
    records.clear();
    NativeLPSZ name = tlNativeLpsz.get();
    int bufSz = metaSize;
    long buf = Unsafe.malloc(bufSz);
    try {
        String base = factory.getConfiguration().getJournalBase().getAbsolutePath();
        Path path = tlPath.get().of(base).$();
        long find = Files.findFirst(path);
        if (find > 0) {
            try {
                long p = -1;
                Path compositePath = tlCompositePath.get();
                do {
                    cancellationHandler.check();
                    if (Files.findType(find) == Files.DT_DIR) {
                        name.of(Files.findName(find));
                        if (Files.isDots(name)) {
                            continue;
                        }
                        long lastModified = Files.getLastModified(compositePath.of(base).concat(name).$());
                        compositePath.of(base).concat(name).concat(JournalConfiguration.FILE_NAME).$();
                        if (Files.exists(compositePath)) {
                            long fd = Files.openRO(compositePath);
                            if (fd < 0) {
                                LOG.error().$("Cannot open: ").$(compositePath).$(" [").$(Os.errno()).$(']').$();
                                continue;
                            }
                            int columnCount;
                            int partitionBy;
                            try {
                                long len = Files.length(compositePath);
                                // don't bother with huge meta files. There is a high chance of them being fake.
                                if (len > maxMetaSize) {
                                    LOG.error().$("File : ").$(compositePath).$(" is too large [").$(len).$(']').$();
                                    continue;
                                }
                                if (len > bufSz) {
                                    Unsafe.free(buf, bufSz);
                                    buf = Unsafe.malloc(bufSz = (int) len);
                                }
                                Files.read(fd, buf, (int) len, 0);
                                // skip over append offset
                                long readPtr = buf + 8;
                                // skip over ID string
                                readPtr += Unsafe.getUnsafe().getInt(readPtr) * 2 + 4;
                                partitionBy = Unsafe.getUnsafe().getInt(readPtr);
                                columnCount = Unsafe.getUnsafe().getInt(readPtr + 4);
                            } finally {
                                Files.close(fd);
                            }
                            p = records.beginRecord(p);
                            // name
                            records.appendStr(name);
                            // partition type
                            records.appendInt(partitionBy);
                            // partition count
                            records.appendInt(countDirs(compositePath.of(base).concat(name).$()));
                            // column count
                            records.appendInt(columnCount);
                            // last modified
                            records.appendLong(lastModified);
                            // size
                            records.appendLong(sumSizes(compositePath));
                        }
                    }
                } while (Files.findNext(find) > 0);
            } finally {
                Files.findClose(find);
            }
        }
    } finally {
        Unsafe.free(buf, bufSz);
    }
    records.toTop();
    return records;
}

Example 4 with NativeLPSZ

use of com.questdb.std.str.NativeLPSZ in project questdb by bluestreak01.

the class FilesTest method testListDir.

@Test
public void testListDir() {
    String temp = temporaryFolder.getRoot().getAbsolutePath();
    ObjList<String> names = new ObjList<>();
    try (Path path = new Path().of(temp).$()) {
        try (Path cp = new Path()) {
            Assert.assertTrue(Files.touch(cp.of(temp).concat("a.txt").$()));
            NativeLPSZ name = new NativeLPSZ();
            long pFind = Files.findFirst(path);
            Assert.assertTrue(pFind != 0);
            try {
                do {
                    names.add(name.of(Files.findName(pFind)).toString());
                } while (Files.findNext(pFind) > 0);
            } finally {
                Files.findClose(pFind);
            }
        }
    }
    names.sort(Chars::compare);
    Assert.assertEquals("[.,..,a.txt]", names.toString());
}

Example 5 with NativeLPSZ

use of com.questdb.std.str.NativeLPSZ in project questdb by bluestreak01.

the class TableWriterTest method testRemoveColumn.

private void testRemoveColumn(TableModel model) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        CairoTestUtils.create(model);
        long ts = DateFormatUtils.parseDateTime("2013-03-04T00:00:00.000Z");
        Rnd rnd = new Rnd();
        try (TableWriter writer = new TableWriter(configuration, model.getName())) {
            // optional
            writer.warmUp();
            ts = append10KProducts(ts, rnd, writer);
            writer.removeColumn("supplier");
            final NativeLPSZ lpsz = new NativeLPSZ();
            try (Path path = new Path()) {
                path.of(root).concat(model.getName());
                final int plen = path.length();
                FF.iterateDir(path.$(), (file, type) -> {
                    lpsz.of(file);
                    if (type == Files.DT_DIR && !Chars.equals(lpsz, '.') && !Chars.equals(lpsz, "..")) {
                        Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.i").$()));
                        Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.d").$()));
                        Assert.assertFalse(FF.exists(path.trimTo(plen).concat(lpsz).concat("supplier.top").$()));
                    }
                });
            }
            ts = append10KNoSupplier(ts, rnd, writer);
            writer.commit();
            Assert.assertEquals(20000, writer.size());
        }
        try (TableWriter writer = new TableWriter(configuration, model.getName())) {
            append10KNoSupplier(ts, rnd, writer);
            writer.commit();
            Assert.assertEquals(30000, writer.size());
        }
    });
}