Examples with Quote org.questdb.examples.support.Quote used on opensource projects

Example 1 with Quote

use of org.questdb.examples.support.Quote in project questdb by bluestreak01.

the class AppendObjectConcurrent method main.

/**
 * Multiple threads append to same journal concurrently.
 * <p>
 * Single append thread receives data from multiple publishers via circular queue using QuestDB's thread messaging mechanism.
 * Please refer to my blog post: http://blog.questdb.org/2016/08/the-art-of-thread-messaging.html for more details.
 */
public static void main(String[] args) throws JournalException, InterruptedException {
    if (args.length != 1) {
        System.out.println("Usage: " + AppendObjectConcurrent.class.getName() + " <path>");
        System.exit(1);
    }
    // 
    RingQueue<Quote> queue = new RingQueue<>(Quote::new, 4096);
    // publisher sequence
    final MPSequence pubSequence = new MPSequence(queue.getCapacity());
    final SCSequence subSequence = new SCSequence();
    // create circular dependency between sequences
    pubSequence.then(subSequence).then(pubSequence);
    // run configuration
    int nThreads = 2;
    int nMessages = 1000000;
    JournalConfiguration configuration = ModelConfiguration.CONFIG.build(args[0]);
    try (Factory factory = new Factory(configuration, 1000, 1, 0)) {
        // start publishing threads
        for (int i = 0; i < nThreads; i++) {
            new Thread(new Publisher(queue, pubSequence, nMessages)).start();
        }
        // consume messages in main thread
        int count = 0;
        int deadline = nMessages * nThreads;
        try (JournalWriter<Quote> writer = factory.writer(Quote.class)) {
            while (count < deadline) {
                long cursor = subSequence.next();
                if (cursor < 0) {
                    LockSupport.parkNanos(1);
                    continue;
                }
                long available = subSequence.available();
                while (cursor < available) {
                    Quote q = queue.get(cursor++);
                    q.setTimestamp(System.currentTimeMillis());
                    try {
                        writer.append(q);
                    } catch (JournalException e) {
                        // append may fail, log and continue
                        // N.B. this logging uses builder pattern to construct message. Building finishes with $() call.
                        LOG.error().$("Sequence failed: ").$(cursor - 1).$(e).$();
                    }
                    count++;
                }
                subSequence.done(available - 1);
                try {
                    writer.commit();
                } catch (JournalException e) {
                    // something serious, attempt to rollback
                    LOG.error().$("Batch commit() failed [").$(available - 1).$(']').$(e).$();
                }
            }
        }
        LOG.info().$("Writer done").$();
    }
}

Example 2 with Quote

use of org.questdb.examples.support.Quote in project questdb by bluestreak01.

the class AppendObjectSortMerge method main.

/**
 * For cases where incoming data feed is not in chronological order but you would like your journal to be in chronological order.
 * This is a lossy way to append data as journal would only be merging a slice of data as specified by "lag" attribute.
 */
public static void main(String[] args) throws JournalException {
    if (args.length != 1) {
        System.out.println("Usage: " + AppendObjectSortMerge.class.getName() + " <path>");
        System.exit(1);
    }
    String journalLocation = args[0];
    try (Factory writerFactory = new Factory(new JournalConfigurationBuilder() {

        {
            $(Quote.class, "quote-lag").lag(24, // enable lag
            TimeUnit.HOURS).$ts();
        }
    }.build(journalLocation), 1000, 1, 0)) {
        // delete existing quote journal
        Files.delete(new File(writerFactory.getConfiguration().getJournalBase(), "quote-lag"));
        try (JournalWriter<Quote> writer = writerFactory.writer(Quote.class)) {
            final String[] symbols = { "AGK.L", "BP.L", "TLW.L", "ABF.L", "LLOY.L", "BT-A.L", "WTB.L", "RRS.L", "ADM.L", "GKN.L", "HSBA.L" };
            final Random r = new Random(System.currentTimeMillis());
            // 20 batches of 50,000 quotes, total 1,000,000
            final int batchCount = 20;
            final int batchSize = 50000;
            final ArrayList<Quote> batch = new ArrayList<>(batchSize);
            // have pre-initialized array to reduce GC overhead
            for (int i = 0; i < batchSize; i++) {
                batch.add(new Quote());
            }
            long utc = System.currentTimeMillis();
            long t = System.nanoTime();
            for (int i = 0; i < batchCount; i++) {
                // populate batch in-memory
                for (int k = 0; k < batchSize; k++) {
                    Quote q = batch.get(k);
                    q.clear();
                    // generate some data
                    q.setSym(symbols[Math.abs(r.nextInt() % (symbols.length - 1))]);
                    q.setAsk(Math.abs(r.nextDouble()));
                    q.setBid(Math.abs(r.nextDouble()));
                    q.setAskSize(Math.abs(r.nextInt() % 10000));
                    q.setBidSize(Math.abs(r.nextInt() % 10000));
                    q.setEx("LXE");
                    q.setMode("Fast trading");
                    long timestamp = utc + (i * batchSize + (batchSize - k)) * 1000L;
                    // make batches overlap (subtract 10 seconds)
                    timestamp -= 100000000L;
                    q.setTimestamp(timestamp);
                }
                // batch must be sorted before being presented to writer
                batch.sort(writer.getTimestampComparator());
                // append batch and have journal merge data
                writer.mergeAppend(batch);
            }
            // commit is necessary
            writer.commit();
            System.out.println("Journal size: " + writer.size());
            System.out.println("Generated " + batchCount * batchSize + " objects in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t) + "ms.");
        }
    }
}

Example 3 with Quote

use of org.questdb.examples.support.Quote in project questdb by bluestreak01.

the class AppendObjectPartitioned method main.

/**
 * Appends 1 million quotes into journal partitioned by day. Journal can only be partitioned on values of timestamp column.
 *
 * @param args factory directory
 * @throws JournalException in case of any problems with the journal.
 */
public static void main(String[] args) throws JournalException {
    if (args.length != 1) {
        System.out.println("Usage: " + AppendObjectPartitioned.class.getName() + " <path>");
        System.exit(1);
    }
    String journalLocation = args[0];
    try (Factory factory = new Factory(new JournalConfigurationBuilder() {

        {
            $(Quote.class, "quote-by-day").partitionBy(PartitionBy.DAY).$ts();
        }
    }.build(journalLocation), 1000, 1, 0)) {
        Files.delete(new File(factory.getConfiguration().getJournalBase(), "quote-by-day"));
        // you can change it in runtime and also, optionally put journal in alternative location
        try (JournalWriter<Quote> writer = factory.writer(Quote.class)) {
            final int count = 1000000;
            final String[] symbols = { "AGK.L", "BP.L", "TLW.L", "ABF.L", "LLOY.L", "BT-A.L", "WTB.L", "RRS.L", "ADM.L", "GKN.L", "HSBA.L" };
            final Rnd r = new Rnd();
            // reuse same same instance of Quote class to keep GC under control
            final Quote q = new Quote();
            long t = System.nanoTime();
            for (int i = 0; i < count; i++) {
                // prepare object for new set of data
                q.clear();
                // generate some data
                q.setSym(symbols[Math.abs(r.nextInt() % (symbols.length - 1))]);
                q.setAsk(Math.abs(r.nextDouble()));
                q.setBid(Math.abs(r.nextDouble()));
                q.setAskSize(Math.abs(r.nextInt() % 10000));
                q.setBidSize(Math.abs(r.nextInt() % 10000));
                q.setEx("LXE");
                q.setMode("Fast trading");
                q.setTimestamp(System.currentTimeMillis() + (i * 100));
                writer.append(q);
            }
            // commit is necessary
            writer.commit();
            System.out.println("Journal size: " + writer.size());
            System.out.println("Generated " + count + " objects in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t) + "ms.");
        }
    }
}

Example 4 with Quote

use of org.questdb.examples.support.Quote in project questdb by bluestreak01.

the class AppendObjectTimeSeries method main.

/**
 * Appends 1 million quotes to journal. Timestamp values are in chronological order.
 */
public static void main(String[] args) throws JournalException {
    if (args.length != 1) {
        System.out.println("Usage: " + AppendObjectTimeSeries.class.getName() + " <path>");
        System.exit(1);
    }
    String journalLocation = args[0];
    JournalConfiguration configuration = new JournalConfigurationBuilder() {

        {
            $(Quote.class, "quote").$ts();
        }
    }.build(journalLocation);
    try (Factory factory = new Factory(configuration, 1000, 1, 0)) {
        // delete existing quote journal
        Files.delete(new File(configuration.getJournalBase(), "quote"));
        try (JournalWriter<Quote> writer = factory.writer(Quote.class)) {
            final int count = 1000000;
            final String[] symbols = { "AGK.L", "BP.L", "TLW.L", "ABF.L", "LLOY.L", "BT-A.L", "WTB.L", "RRS.L", "ADM.L", "GKN.L", "HSBA.L" };
            final Random r = new Random(System.currentTimeMillis());
            // reuse same same instance of Quote class to keep GC under control
            final Quote q = new Quote();
            long t = System.nanoTime();
            for (int i = 0; i < count; i++) {
                // prepare object for new set of data
                q.clear();
                // generate some data
                q.setSym(symbols[Math.abs(r.nextInt() % (symbols.length - 1))]);
                q.setAsk(Math.abs(r.nextDouble()));
                q.setBid(Math.abs(r.nextDouble()));
                q.setAskSize(Math.abs(r.nextInt() % 10000));
                q.setBidSize(Math.abs(r.nextInt() % 10000));
                q.setEx("LXE");
                q.setMode("Fast trading");
                q.setTimestamp(System.currentTimeMillis());
                writer.append(q);
            }
            // commit is necessary
            writer.commit();
            System.out.println("Journal size: " + writer.size());
            System.out.println("Generated " + count + " objects in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t) + "ms.");
        }
    }
}

Example 5 with Quote

use of org.questdb.examples.support.Quote in project questdb by bluestreak01.

the class AppendObjectUnordered method main.

/**
 * Appends 1 million quotes with random timestamp values. Journal doesn't enforce order.
 */
public static void main(String[] args) throws JournalException {
    if (args.length != 1) {
        System.out.println("Usage: " + AppendObjectTimeSeries.class.getName() + " <path>");
        System.exit(1);
    }
    String journalLocation = args[0];
    try (Factory factory = new Factory(journalLocation, 1000, 1, 0)) {
        // delete existing quote journal
        Files.delete(new File(factory.getConfiguration().getJournalBase(), "quote-unordered"));
        try (JournalWriter<Quote> writer = factory.writer(Quote.class)) {
            final int count = 1000000;
            final String[] symbols = { "AGK.L", "BP.L", "TLW.L", "ABF.L", "LLOY.L", "BT-A.L", "WTB.L", "RRS.L", "ADM.L", "GKN.L", "HSBA.L" };
            final Random r = new Random(System.currentTimeMillis());
            // reuse same same instance of Quote class to keep GC under control
            final Quote q = new Quote();
            long t = System.nanoTime();
            for (int i = 0; i < count; i++) {
                // prepare object for new set of data
                q.clear();
                // generate some data
                q.setSym(symbols[Math.abs(r.nextInt() % (symbols.length - 1))]);
                q.setAsk(Math.abs(r.nextDouble()));
                q.setBid(Math.abs(r.nextDouble()));
                q.setAskSize(Math.abs(r.nextInt() % 10000));
                q.setBidSize(Math.abs(r.nextInt() % 10000));
                q.setEx("LXE");
                q.setMode("Fast trading");
                q.setTimestamp(r.nextLong());
                // 
                writer.append(q);
            }
            // commit is necessary
            writer.commit();
            System.out.println("Journal size: " + writer.size());
            System.out.println("Generated " + count + " objects in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t) + "ms.");
        }
    }
}