Examples with ChronicleQueue net.openhft.chronicle.queue.ChronicleQueue used on opensource projects

Example 66 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class TailerTest method reproduce.

@Test
public void reproduce() {
    IOTools.deleteDirWithFiles(QUEUE_PATH.toFile());
    long firstOutputIndex = Long.MAX_VALUE;
    long lastOutputIndex = Long.MIN_VALUE;
    try (ChronicleQueue q = createQueue();
        ExcerptAppender appender = q.acquireAppender();
        ExcerptTailer tailer = q.createTailer()) {
        for (int i = 0; i < 5; i++) {
            final String text = "Hello World " + i;
            try (DocumentContext dc = appender.writingDocument()) {
                dc.wire().writeText(text);
            }
            System.out.format("Appended \"%s\" at %d%n", text, appender.lastIndexAppended());
            firstOutputIndex = Math.min(firstOutputIndex, appender.lastIndexAppended());
            lastOutputIndex = Math.max(lastOutputIndex, appender.lastIndexAppended());
        }
        System.out.format("firstOutputIndex = %d%n", firstOutputIndex);
        System.out.format("lastOutputIndex = %d%n", lastOutputIndex);
        System.out.println("Reading initial");
        drainTailer(tailer);
    }
    try (ChronicleQueue q = createQueue();
        ExcerptTailer tailer = q.createTailer()) {
        initRecovery(tailer, firstOutputIndex + OFFSET);
        final List<String> messages = drainTailer(tailer);
        assertEquals("Hello World " + OFFSET, messages.get(0));
    }
}

Example 67 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class LatencyDistributionMain method run.

public void run(String[] args) throws InterruptedException {
    File tmpDir = getTmpDir();
    SingleChronicleQueueBuilder builder = SingleChronicleQueueBuilder.fieldlessBinary(tmpDir).blockSize(128 << 20);
    try (ChronicleQueue queue = builder.writeBufferMode(BUFFER_MODE).readBufferMode(BufferMode.None).build();
        ChronicleQueue queue2 = builder.writeBufferMode(BufferMode.None).readBufferMode(BUFFER_MODE).build()) {
        runTest(queue, queue2);
    }
    IOTools.deleteDirWithFiles(tmpDir, 2);
}

Example 68 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class LatencyDistributionMain method runTest.

protected void runTest(@NotNull ChronicleQueue queue, @NotNull ChronicleQueue queue2) throws InterruptedException {
    Histogram histogramCo = new Histogram();
    Histogram histogramIn = new Histogram();
    Histogram histogramWr = new Histogram();
    Thread pretoucher = new Thread(() -> {
        ExcerptAppender appender = queue.acquireAppender();
        try {
            while (!Thread.currentThread().isInterrupted()) {
                appender.pretouch();
                Jvm.pause(50);
            }
        } catch (Exception e) {
            if (!appender.isClosed())
                e.printStackTrace();
        }
    });
    pretoucher.setDaemon(true);
    pretoucher.start();
    ExcerptAppender appender = queue.acquireAppender();
    // two queues as most like in a different process.
    ExcerptTailer tailer = queue2.createTailer();
    String name = getClass().getName();
    Thread tailerThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableTailerAffinity") || !Jvm.getBoolean("disableAffinity")) {
                lock = Affinity.acquireLock();
            }
            int counter = 0;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    // if (SAMPLING)
                    // sampler.thread(Thread.currentThread());
                    // boolean found = tailer.readDocument(myReadMarshallable);
                    boolean found;
                    try (DocumentContext dc = tailer.readingDocument()) {
                        found = dc.isPresent();
                        if (found) {
                            int count = counter++;
                            if (count == WARMUP) {
                                histogramCo.reset();
                                histogramIn.reset();
                                histogramWr.reset();
                            }
                            Bytes<?> bytes = dc.wire().bytes();
                            long startCo = bytes.readLong();
                            long startIn = bytes.readLong();
                            long now = System.nanoTime();
                            histogramCo.sample(now - startCo);
                            histogramIn.sample(now - startIn);
                            if (count % INTLOG_INTERVAL == 0)
                                System.out.println("read  " + count);
                        }
                    }
                /*
                        if (SAMPLING) {
                            StackTraceElement[] stack = sampler.getAndReset();
                            if (stack != null) {
                                if (!stack[0].getClassName().equals(name) &&
                                        !stack[0].getClassName().equals("java.lang.Thread")) {
                                    StringBuilder sb = new StringBuilder();
                                    Jvm.trimStackTrace(sb, stack);
                                   // System.out.println(sb);
                                }
                            } else if (!found) {
                                Thread.yield();
                            }
                        }
                        */
                } catch (Exception e) {
                    break;
                }
            }
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    Thread appenderThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableAppenderAffinity") || !Jvm.getBoolean("disableAffinity")) {
                lock = Affinity.acquireLock();
            }
            long next = System.nanoTime();
            long interval = 1_000_000_000 / throughput;
            Map<String, Integer> stackCount = new LinkedHashMap<>();
            BytesStore<?, ?> bytes24 = BytesStore.nativeStoreFrom(new byte[Main.size - 16]);
            for (int i = -WARMUP; i < iterations; i++) {
                long s0 = System.nanoTime();
                if (s0 < next) {
                    do ; while (System.nanoTime() < next);
                    // if we failed to come out of the spin loop on time, reset next.
                    next = System.nanoTime();
                }
                if (SAMPLING) {
                    sampler.thread(Thread.currentThread());
                }
                long start = System.nanoTime();
                try (@NotNull DocumentContext dc = appender.writingDocument(false)) {
                    Wire wire = dc.wire();
                    Bytes<?> bytes2 = wire.bytes();
                    // when it should have started
                    bytes2.writeLong(next);
                    // when it actually started.
                    bytes2.writeLong(start);
                    bytes2.write(bytes24);
                    ThroughputMain.addToEndOfCache(wire);
                }
                long time = System.nanoTime() - start;
                histogramWr.sample(start - next);
                if (SAMPLING && time > 1e3 && i > 0) {
                    StackTraceElement[] stack = sampler.getAndReset();
                    if (stack != null) {
                        if (!stack[0].getClassName().equals(name) && !stack[0].getClassName().equals("java.lang.Thread")) {
                            StringBuilder sb = new StringBuilder();
                            Jvm.trimStackTrace(sb, stack);
                            stackCount.compute(sb.toString(), (k, v) -> v == null ? 1 : v + 1);
                        }
                    }
                }
                next += interval;
                if (i % INTLOG_INTERVAL == 0)
                    System.out.println("wrote " + i);
            }
            stackCount.entrySet().stream().filter(e -> e.getValue() > 1).forEach(System.out::println);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    tailerThread.start();
    appenderThread.start();
    appenderThread.join();
    pretoucher.interrupt();
    pretoucher.join();
    // Pause to allow tailer to catch up (if needed)
    Jvm.pause(500);
    tailerThread.interrupt();
    tailerThread.join();
    System.out.println("wr: " + histogramWr.toLongMicrosFormat());
    System.out.println("in: " + histogramIn.toLongMicrosFormat());
    System.out.println("co: " + histogramCo.toLongMicrosFormat());
}

Example 69 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class SingleChronicleQueueTest method testSingleWire.

// *************************************************************************
// 
// *************************************************************************
@Test
public void testSingleWire() {
    final File file = createTempFile("testSingleWire");
    try {
        final ChronicleQueue chronicle = createQueue(file);
        final ExcerptAppender appender = chronicle.acquireAppender();
        appender.writeDocument(wire -> wire.write("FirstName").text("Steve"));
        appender.writeDocument(wire -> wire.write("Surname").text("Jobs"));
        StringBuilder first = new StringBuilder();
        StringBuilder surname = new StringBuilder();
        final ExcerptTailer tailer = chronicle.createTailer();
        tailer.readDocument(wire -> wire.read("FirstName").text(first));
        tailer.readDocument(wire -> wire.read("Surname").text(surname));
        Assert.assertEquals("Steve Jobs", first + " " + surname);
    } finally {
        file.delete();
    }
}

Example 70 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class ChronicleReader method execute.

public void execute() {
    long lastObservedTailIndex;
    long highestReachedIndex = 0L;
    boolean isFirstIteration = true;
    boolean retryLastOperation;
    boolean queueHasBeenModified;
    final AtomicLong matchCounter = new AtomicLong(0L);
    do {
        try (final ChronicleQueue queue = createQueue();
            final QueueEntryHandler messageConverter = entryHandlerFactory.get()) {
            final ExcerptTailer tailer = queue.createTailer();
            MessageHistory.set(new VanillaMessageHistory());
            tlTailer = ThreadLocal.withInitial(queue::createTailer);
            do {
                if (highestReachedIndex != 0L) {
                    tailer.moveToIndex(highestReachedIndex);
                }
                final Bytes textConversionTarget = Bytes.elasticByteBuffer();
                try {
                    moveToSpecifiedPosition(queue, tailer, isFirstIteration);
                    lastObservedTailIndex = tailer.index();
                    Consumer<String> messageConsumer = text -> applyFiltersAndLog(text, tailer.index(), matchCounter);
                    BooleanSupplier readOne;
                    if (methodReaderInterface == null) {
                        readOne = () -> readOne(messageConverter, tailer, messageConsumer);
                    } else {
                        // TODO: consider unifying this with messageConverter
                        Bytes<ByteBuffer> bytes = Bytes.elasticHeapByteBuffer(256);
                        Wire wire = wireType.apply(bytes);
                        if (wire instanceof TextWire)
                            ((TextWire) wire).useTextDocuments();
                        MethodWriterBuilder<?> mwb = wire.methodWriterBuilder(methodReaderInterface);
                        if (showMessageHistory)
                            mwb.updateInterceptor((methodName, t) -> {
                                MessageHistory messageHistory = MessageHistory.get();
                                // this is an attempt to recognise that no MH was read and instead the method reader called reset(...) on it
                                if (messageHistory.sources() != 1 || messageHistory.timings() != 1)
                                    bytes.append(messageHistory + System.lineSeparator());
                                return true;
                            });
                        MethodReader methodReader = tailer.methodReader(mwb.build());
                        readOne = () -> {
                            boolean found = methodReader.readOne();
                            if (found)
                                messageConsumer.accept(bytes.toString());
                            bytes.clear();
                            return found;
                        };
                    }
                    while (!Thread.currentThread().isInterrupted()) {
                        boolean found = readOne.getAsBoolean();
                        if (!found) {
                            if (tailInputSource) {
                                pauser.pause();
                            }
                            break;
                        } else {
                            if (matchLimitReached(matchCounter.get())) {
                                break;
                            }
                        }
                        pauser.reset();
                    }
                } finally {
                    textConversionTarget.releaseLast();
                    highestReachedIndex = tailer.index();
                    isFirstIteration = false;
                }
                queueHasBeenModified = queueHasBeenModifiedSinceLastCheck(lastObservedTailIndex);
                retryLastOperation = false;
                if (!running || matchLimitReached(matchCounter.get()))
                    return;
            } while (tailerDirection != BACKWARD && (tailInputSource || queueHasBeenModified));
        } catch (final RuntimeException e) {
            if (e.getCause() != null && e.getCause() instanceof DateTimeParseException) {
                // ignore this error - due to a race condition between
                // the reader creating a Queue (with default roll-cycle due to no files on disk)
                // and the writer appending to the Queue with a non-default roll-cycle
                retryLastOperation = true;
            } else {
                throw e;
            }
        }
    } while (retryLastOperation);
}