Examples with Path io.questdb.std.str.Path used on opensource projects

Example 91 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class SymbolCacheTest method testConcurrency.

@Test
public void testConcurrency() throws Exception {
    assertMemoryLeak(() -> {
        final Rnd rndCache = new Rnd();
        final int N = 1_000_000;
        long ts = TimestampFormatUtils.parseTimestamp("2020-09-10T20:00:00.000000Z");
        final long incrementUs = 10000;
        final String constValue = "hello";
        compiler.compile("create table x(a symbol, c int, b symbol capacity 10000000, ts timestamp) timestamp(ts) partition by DAY", sqlExecutionContext);
        try (SymbolCache symbolCache = new SymbolCache(new DefaultLineTcpReceiverConfiguration());
            Path path = new Path()) {
            path.of(configuration.getRoot()).concat("x");
            symbolCache.of(configuration, path, "b", 1);
            final CyclicBarrier barrier = new CyclicBarrier(2);
            final SOCountDownLatch haltLatch = new SOCountDownLatch(1);
            final AtomicBoolean cacheInError = new AtomicBoolean(false);
            RingQueue<Holder> wheel = new RingQueue<Holder>(Holder::new, 256);
            SPSequence pubSeq = new SPSequence(wheel.getCycle());
            SCSequence subSeq = new SCSequence();
            pubSeq.then(subSeq).then(pubSeq);
            new Thread(() -> {
                try {
                    barrier.await();
                    for (int i = 0; i < N; i++) {
                        // All keys should not be found, but we keep looking them up because
                        // we pretend we don't know this upfront. The aim is to cause
                        // race condition between lookup and table writer
                        final CharSequence value2 = rndCache.nextString(5);
                        symbolCache.getSymbolKey(constValue);
                        symbolCache.getSymbolKey(value2);
                        final long cursor = pubSeq.nextBully();
                        final Holder h = wheel.get(cursor);
                        // publish the value2 to the table writer
                        h.value1 = constValue;
                        h.value2 = Chars.toString(value2);
                        pubSeq.done(cursor);
                    }
                } catch (Throwable e) {
                    cacheInError.set(true);
                    e.printStackTrace();
                } finally {
                    haltLatch.countDown();
                }
            }).start();
            try (TableWriter w = engine.getWriter(sqlExecutionContext.getCairoSecurityContext(), "x", "test")) {
                barrier.await();
                OUT: for (int i = 0; i < N; i++) {
                    long cursor;
                    while (true) {
                        cursor = subSeq.next();
                        if (cursor < 0) {
                            // due to random generator producing duplicate strings
                            if (haltLatch.getCount() < 1) {
                                break OUT;
                            }
                        } else {
                            break;
                        }
                    }
                    Holder h = wheel.get(cursor);
                    TableWriter.Row r = w.newRow(ts);
                    r.putSym(0, h.value1);
                    r.putInt(1, 0);
                    r.putSym(2, h.value2);
                    r.append();
                    subSeq.done(cursor);
                    if (i % 256 == 0) {
                        w.commit();
                    }
                    ts += incrementUs;
                }
                w.commit();
            } finally {
                haltLatch.await();
            }
            Assert.assertFalse(cacheInError.get());
        }
        compiler.compile("drop table x", sqlExecutionContext);
    });
}

Example 92 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class SymbolCacheTest method testSimpleInteraction.

@Test
public void testSimpleInteraction() throws Exception {
    String tableName = "tb1";
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path();
            TableModel model = new TableModel(configuration, tableName, PartitionBy.DAY).col("symCol1", ColumnType.SYMBOL).col("symCol2", ColumnType.SYMBOL);
            SymbolCache cache = new SymbolCache(new DefaultLineTcpReceiverConfiguration() {

                @Override
                public long getSymbolCacheWaitUsBeforeReload() {
                    return 0;
                }
            })) {
            CairoTestUtils.create(model);
            try (TableWriter writer = new TableWriter(configuration, tableName);
                MemoryMR txMem = Vm.getMRInstance()) {
                int symColIndex1 = writer.getColumnIndex("symCol1");
                int symColIndex2 = writer.getColumnIndex("symCol2");
                long symCountOffset = TableUtils.getSymbolWriterIndexOffset(symColIndex2);
                long transientSymCountOffset = TableUtils.getSymbolWriterTransientIndexOffset(symColIndex2);
                path.of(configuration.getRoot()).concat(tableName);
                txMem.of(configuration.getFilesFacade(), path.concat(TableUtils.TXN_FILE_NAME).$(), transientSymCountOffset + Integer.BYTES, transientSymCountOffset + Integer.BYTES, MemoryTag.MMAP_DEFAULT);
                cache.of(configuration, path.of(configuration.getRoot()).concat(tableName), "symCol2", symColIndex2);
                TableWriter.Row r = writer.newRow();
                r.putSym(symColIndex1, "sym11");
                r.putSym(symColIndex2, "sym21");
                r.append();
                writer.commit();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(1, txMem.getInt(transientSymCountOffset));
                int rc = cache.getSymbolKey("missing");
                Assert.assertEquals(SymbolTable.VALUE_NOT_FOUND, rc);
                Assert.assertEquals(0, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym21");
                r.append();
                writer.commit();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(1, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("missing");
                Assert.assertEquals(SymbolTable.VALUE_NOT_FOUND, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym22");
                r.append();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(2, txMem.getInt(transientSymCountOffset));
                writer.commit();
                Assert.assertEquals(2, txMem.getInt(symCountOffset));
                Assert.assertEquals(2, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym22");
                Assert.assertEquals(1, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                // Test cached uncommitted symbols
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym23");
                r.append();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym24");
                r.append();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym25");
                r.append();
                Assert.assertEquals(2, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym22");
                Assert.assertEquals(1, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym24");
                Assert.assertEquals(3, rc);
                Assert.assertEquals(3, cache.getCacheValueCount());
                writer.commit();
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                // Test deleting a symbol column
                writer.removeColumn("symCol1");
                cache.close();
                txMem.close();
                symColIndex2 = writer.getColumnIndex("symCol2");
                symCountOffset = TableUtils.getSymbolWriterIndexOffset(symColIndex2);
                transientSymCountOffset = TableUtils.getSymbolWriterTransientIndexOffset(symColIndex2);
                path.of(configuration.getRoot()).concat(tableName);
                txMem.of(configuration.getFilesFacade(), path.concat(TableUtils.TXN_FILE_NAME).$(), transientSymCountOffset + Integer.BYTES, transientSymCountOffset + Integer.BYTES, MemoryTag.MMAP_DEFAULT);
                cache.of(configuration, path.of(configuration.getRoot()).concat(tableName), "symCol2", symColIndex2);
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym24");
                Assert.assertEquals(3, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex2, "sym26");
                r.append();
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(6, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym26");
                Assert.assertEquals(5, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                writer.commit();
                Assert.assertEquals(6, txMem.getInt(symCountOffset));
                Assert.assertEquals(6, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym26");
                Assert.assertEquals(5, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
            }
        }
    });
}

Example 93 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class LineUdpParserImplTest method testCannotCreateTable.

@Test
public void testCannotCreateTable() throws Exception {
    TestFilesFacade ff = new TestFilesFacade() {

        boolean called = false;

        @Override
        public int mkdirs(LPSZ path, int mode) {
            if (Chars.endsWith(path, "x" + Files.SEPARATOR)) {
                called = true;
                return -1;
            }
            return super.mkdirs(path, mode);
        }

        @Override
        public boolean wasCalled() {
            return called;
        }
    };
    final String expected = "sym\tdouble\tint\tbool\tstr\ttimestamp\n" + "zzz\t1.3\t11\tfalse\tnice\t2017-10-03T10:00:00.000000Z\n";
    String lines = "x,sym2=xyz double=1.6,int=15i,bool=true,str=\"string1\"\n" + "x,sym1=abc double=1.3,int=11i,bool=false,str=\"string2\"\n" + "y,sym=zzz double=1.3,int=11i,bool=false,str=\"nice\"\n";
    CairoConfiguration configuration = new DefaultCairoConfiguration(root) {

        @Override
        public FilesFacade getFilesFacade() {
            return ff;
        }

        @Override
        public MicrosecondClock getMicrosecondClock() {
            try {
                return new TestMicroClock(TimestampFormatUtils.parseTimestamp("2017-10-03T10:00:00.000Z"), 10);
            } catch (NumericException e) {
                throw new RuntimeException(e);
            }
        }
    };
    assertThat(expected, lines, "y", configuration);
    Assert.assertTrue(ff.wasCalled());
    try (Path path = new Path()) {
        Assert.assertEquals(TableUtils.TABLE_DOES_NOT_EXIST, TableUtils.exists(ff, path, root, "all"));
    }
}

Example 94 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class MimeTypesCacheTest method testCannotOpen.

@Test()
public void testCannotOpen() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path()) {
            path.of("/tmp/sdrqwhlkkhlkhasdlkahdoiquweoiuweoiqwe.ok").$();
            try {
                new MimeTypesCache(FilesFacadeImpl.INSTANCE, path);
                Assert.fail();
            } catch (HttpException e) {
                Assert.assertTrue(Chars.startsWith(e.getMessage(), "could not open"));
            }
        }
    });
}

Example 95 with Path

use of io.questdb.std.str.Path in project questdb by bluestreak01.

the class MimeTypesCacheTest method testFailure.

private void testFailure(FilesFacade ff, CharSequence startsWith) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path()) {
            try {
                new MimeTypesCache(ff, path);
                Assert.fail();
            } catch (HttpException e) {
                Assert.assertTrue(Chars.startsWith(e.getMessage(), startsWith));
            }
        }
    });
}