Examples with JournalMetadata com.questdb.store.factory.configuration.JournalMetadata used on opensource projects

Example 26 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class TableUtilsTest method testCreate.

@Test
public void testCreate() {
    final CharSequence root = temp.getRoot().getAbsolutePath();
    final JournalMetadata metadata = getJournalStructure().build();
    try (AppendMemory appendMemory = new AppendMemory()) {
        try (Path path = new Path()) {
            TableUtils.create(FF, path, appendMemory, root, metadata, 509);
            Assert.assertEquals(TABLE_EXISTS, TableUtils.exists(FF, path, root, metadata.getName()));
            path.of(root).concat(metadata.getName()).concat("_meta").$();
            try (ReadOnlyMemory mem = new ReadOnlyMemory(FF, path, Files.PAGE_SIZE, FF.length(path))) {
                long p = 0;
                Assert.assertEquals(metadata.getColumnCount(), mem.getInt(p));
                p += 4;
                Assert.assertEquals(PartitionBy.NONE, mem.getInt(p));
                p += 4;
                Assert.assertEquals(metadata.getTimestampIndex(), mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.INT, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.DOUBLE, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.FLOAT, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.BYTE, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.LONG, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.STRING, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.BOOLEAN, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.SYMBOL, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.SHORT, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.DATE, mem.getInt(p));
                p += 4;
                Assert.assertEquals(ColumnType.DATE, mem.getInt(p));
                p += 4;
                p = assertCol(mem, p, "i");
                p = assertCol(mem, p, "d");
                p = assertCol(mem, p, "f");
                p = assertCol(mem, p, "b");
                p = assertCol(mem, p, "l");
                p = assertCol(mem, p, "str");
                p = assertCol(mem, p, "boo");
                p = assertCol(mem, p, "sym");
                p = assertCol(mem, p, "sho");
                p = assertCol(mem, p, "date");
                assertCol(mem, p, "timestamp");
            }
        }
    }
}

Example 27 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class JournalMetadataTest method testMetadataWrite.

@Test
public void testMetadataWrite() throws Exception {
    JournalMetadataBuilder<Quote> b = new JournalMetadataBuilder<>(Quote.class);
    UnstructuredFile hb = new UnstructuredFile(temp.newFile(), 10, JournalMode.APPEND);
    JournalMetadata m = b.build();
    m.write(hb);
    JournalMetadata metadata = new JournalMetadata(hb, null);
    hb.close();
    Assert.assertTrue(m.isCompatible(metadata, false));
}

Example 28 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class CachingReaderFactoryTest method testLockBusyReader.

@Test
public void testLockBusyReader() throws Exception {
    final int readerCount = 5;
    int threadCount = 2;
    final int iterations = 10000;
    // create journals to read
    final JournalMetadata<?>[] meta = new JournalMetadata[readerCount];
    for (int i = 0; i < readerCount; i++) {
        final JournalMetadata<?> m = new JournalStructure("x" + i).$date("ts").$().build();
        ((WriterFactory) getFactory()).writer(m).close();
        meta[i] = m;
    }
    try {
        try (final CachingReaderFactory rf = new CachingReaderFactory(factoryContainer.getConfiguration(), 1000, 2)) {
            final CyclicBarrier barrier = new CyclicBarrier(threadCount);
            final CountDownLatch halt = new CountDownLatch(threadCount);
            final AtomicInteger errors = new AtomicInteger();
            final LongList lockTimes = new LongList();
            final LongList workerTimes = new LongList();
            new Thread(() -> {
                Rnd rnd = new Rnd();
                try {
                    barrier.await();
                    String name = null;
                    for (int i = 0; i < iterations; i++) {
                        if (name == null) {
                            name = meta[rnd.nextPositiveInt() % readerCount].getName();
                        }
                        while (true) {
                            try {
                                rf.lock(name);
                                lockTimes.add(System.currentTimeMillis());
                                LockSupport.parkNanos(100L);
                                rf.unlock(name);
                                name = null;
                                break;
                            } catch (JournalException e) {
                                if (!(e instanceof RetryLockException)) {
                                    e.printStackTrace();
                                    errors.incrementAndGet();
                                    break;
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    errors.incrementAndGet();
                }
                halt.countDown();
            }).start();
            new Thread(() -> {
                Rnd rnd = new Rnd();
                workerTimes.add(System.currentTimeMillis());
                for (int i = 0; i < iterations; i++) {
                    JournalMetadata<?> metadata = meta[rnd.nextPositiveInt() % readerCount];
                    try (Journal<?> ignored = rf.reader(metadata)) {
                        if (metadata == meta[readerCount - 1] && barrier.getNumberWaiting() > 0) {
                            barrier.await();
                        }
                        LockSupport.parkNanos(10L);
                    } catch (JournalLockedException ignored) {
                    } catch (Exception e) {
                        e.printStackTrace();
                        errors.incrementAndGet();
                    }
                }
                workerTimes.add(System.currentTimeMillis());
                halt.countDown();
            }).start();
            halt.await();
            Assert.assertEquals(0, errors.get());
            // check that there are lock times between worker times
            int count = 0;
            // ensure that we have worker times
            Assert.assertEquals(2, workerTimes.size());
            long lo = workerTimes.get(0);
            long hi = workerTimes.get(1);
            Assert.assertTrue(lockTimes.size() > 0);
            for (int i = 0, n = lockTimes.size(); i < n; i++) {
                long t = lockTimes.getQuick(i);
                if (t > lo && t < hi) {
                    count++;
                }
            }
            Assert.assertTrue(count > 0);
        }
    } catch (Throwable e) {
        e.printStackTrace();
    }
}

Example 29 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class JournalClient method subscribeOne.

private void subscribeOne(int index, SubscriptionHolder holder, String name, boolean newSubscription) {
    if (newSubscription) {
        SubscriptionHolder sub = new SubscriptionHolder();
        sub.local = holder.local;
        sub.remote = holder.remote;
        sub.listener = holder.listener;
        sub.writer = holder.writer;
        subscriptions.add(sub);
    }
    JournalWriter<?> writer = writers.getQuiet(index);
    try {
        commandProducer.write(channel, Command.ADD_KEY_CMD);
        setKeyRequestProducer.write(channel, new IndexedJournalKey(index, holder.remote));
        checkAck();
        // todo: do we really have to use file here?
        JournalMetadata<?> metadata;
        File file = Files.makeTempFile();
        try {
            try (HugeBufferConsumer h = new HugeBufferConsumer(file)) {
                h.read(channel);
                metadata = new JournalMetadata(h.getHb(), name);
            } catch (JournalException e) {
                throw new JournalNetworkException(e);
            }
        } finally {
            Files.delete(file);
        }
        boolean validate = true;
        if (writer == null) {
            if (holder.writer == null) {
                try {
                    writer = factory.writer(metadata);
                } catch (JournalException e) {
                    LOG.error().$("Failed to create writer: ").$(e).$();
                    unsubscribe(index, null, holder, JournalEvents.EVT_JNL_INCOMPATIBLE);
                    return;
                }
                writersToClose.add(writer);
                validate = false;
            } else {
                writer = holder.writer;
            }
            writer.disableCommitOnClose();
            statusSentList.extendAndSet(index, 0);
            deltaConsumers.extendAndSet(index, new JournalDeltaConsumer(writer));
            writers.extendAndSet(index, writer);
            writer.setJournalListener(holder.listener);
        } else {
            statusSentList.setQuick(index, 0);
        }
        if (validate && !metadata.isCompatible(writer.getMetadata(), false)) {
            LOG.error().$("Journal ").$(holder.local.getName()).$(" is not compatible with ").$(holder.remote.getName()).$("(remote)").$();
            unsubscribe(index, writer, holder, JournalEvents.EVT_JNL_INCOMPATIBLE);
            return;
        }
        commandProducer.write(channel, Command.DELTA_REQUEST_CMD);
        journalClientStateProducer.write(channel, new IndexedJournal(index, writer));
        checkAck();
        statusSentList.setQuick(index, 1);
        if (holder.listener != null) {
            holder.listener.onEvent(JournalEvents.EVT_JNL_SUBSCRIBED);
        }
        LOG.info().$("Subscribed ").$(name).$(" to ").$(holder.remote.getName()).$("(remote)").$();
    } catch (JournalNetworkException e) {
        LOG.error().$("Failed to subscribe ").$(name).$(" to ").$(holder.remote.getName()).$("(remote)").$();
        unsubscribe(index, writer, holder, JournalEvents.EVT_JNL_SERVER_ERROR);
    }
}