Examples with BytesStore net.openhft.chronicle.bytes.BytesStore used on opensource projects

Example 1 with BytesStore

use of net.openhft.chronicle.bytes.BytesStore in project Chronicle-Bytes by OpenHFT.

the class AbstractInterner method intern.

public T intern(@NotNull BytesStore cs, int length) throws IllegalArgumentException, IORuntimeException, BufferUnderflowException {
    if (length > entries.length)
        return getValue(cs, length);
    int hash = hash32(cs, length);
    int h = hash & mask;
    InternerEntry<T> s = entries[h];
    if (s != null && s.bytes.equalBytes(cs, length))
        return s.t;
    int h2 = (hash >> shift) & mask;
    InternerEntry<T> s2 = entries[h2];
    if (s2 != null && s2.bytes.equalBytes(cs, length))
        return s2.t;
    @NotNull T t = getValue(cs, length);
    @NotNull final byte[] bytes = new byte[length];
    @NotNull BytesStore bs = BytesStore.wrap(bytes);
    cs.read(cs.readPosition(), bytes, 0, length);
    entries[s == null || (s2 != null && toggle()) ? h : h2] = new InternerEntry<>(bs, t);
    return t;
}

Example 2 with BytesStore

use of net.openhft.chronicle.bytes.BytesStore in project Chronicle-Queue by OpenHFT.

the class ChronicleQueueLatencyDistribution method runTest.

protected void runTest(@NotNull ChronicleQueue queue, int throughput) throws InterruptedException {
    /*
        Jvm.setExceptionHandlers(PrintExceptionHandler.ERR,
                PrintExceptionHandler.OUT,
                PrintExceptionHandler.OUT);
*/
    Histogram histogramCo = new Histogram();
    Histogram histogramIn = new Histogram();
    Histogram histogramWr = new Histogram();
    if (PRETOUCH) {
        Thread pretoucher = new Thread(() -> {
            ExcerptAppender appender = queue.acquireAppender();
            while (!Thread.currentThread().isInterrupted()) {
                appender.pretouch();
                Jvm.pause(500);
            }
        });
        pretoucher.setName("pret");
        pretoucher.setDaemon(true);
        pretoucher.start();
    }
    ExcerptAppender appender = queue.acquireAppender();
    ExcerptTailer tailer = queue.createTailer();
    String name = getClass().getName();
    Thread tailerThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableTailerAffinity") || Jvm.getBoolean("enableAffinity")) {
                lock = Affinity.acquireLock();
            }
            int counter = 0;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    // if (SAMPLING)
                    // sampler.thread(Thread.currentThread());
                    // boolean found = tailer.readDocument(myReadMarshallable);
                    boolean found;
                    try (DocumentContext dc = tailer.readingDocument()) {
                        found = dc.isPresent();
                        if (found) {
                            int count = counter++;
                            if (count == WARMUP) {
                                histogramCo.reset();
                                histogramIn.reset();
                                histogramWr.reset();
                            }
                            Bytes<?> bytes = dc.wire().bytes();
                            long startCo = bytes.readLong();
                            long startIn = bytes.readLong();
                            long now = System.nanoTime();
                            histogramCo.sample(now - startCo);
                            histogramIn.sample(now - startIn);
                        // if (count % 1_000_000 == 0) System.out.println("read  " + count);
                        }
                    }
                /*
                        if (SAMPLING) {
                            StackTraceElement[] stack = sampler.getAndReset();
                            if (stack != null) {
                                if (!stack[0].getClassName().equals(name) &&
                                        !stack[0].getClassName().equals("java.lang.Thread")) {
                                    StringBuilder sb = new StringBuilder();
                                    Jvm.trimStackTrace(sb, stack);
                                   // System.out.println(sb);
                                }
                            } else if (!found) {
                                Thread.yield();
                            }
                        }
                        */
                } catch (Exception e) {
                    break;
                }
            }
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    Thread appenderThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableAppenderAffinity") || Jvm.getBoolean("enableAffinity")) {
                lock = Affinity.acquireLock();
            }
            long next = System.nanoTime();
            long interval = 1_000_000_000 / throughput;
            Map<String, Integer> stackCount = new LinkedHashMap<>();
            BytesStore bytes24 = BytesStore.nativeStore(24);
            for (int i = -WARMUP; i < ITERATIONS; i++) {
                long s0 = System.nanoTime();
                if (s0 < next) {
                    do ; while (System.nanoTime() < next);
                    // if we failed to come out of the spin loop on time, reset next.
                    next = System.nanoTime();
                }
                if (SAMPLING) {
                    sampler.thread(Thread.currentThread());
                }
                long start = System.nanoTime();
                try (@NotNull DocumentContext dc = appender.writingDocument(false)) {
                    Bytes<?> bytes2 = dc.wire().bytes();
                    // when it should have started
                    bytes2.writeLong(next);
                    // when it actually started.
                    bytes2.writeLong(start);
                    bytes2.write(bytes24);
                }
                long time = System.nanoTime() - start;
                histogramWr.sample(start - next);
                if (SAMPLING && time > SAMPLE_THRESHOLD_NS && i > 0) {
                    StackTraceElement[] stack = sampler.getAndReset();
                    if (stack != null) {
                        if (!stack[0].getClassName().equals(name) && !stack[0].getClassName().equals("java.lang.Thread")) {
                            StringBuilder sb = new StringBuilder();
                            Jvm.trimStackTrace(sb, stack);
                            stackCount.compute(sb.toString(), (k, v) -> v == null ? 1 : v + 1);
                        }
                    }
                }
                next += interval;
            // if (i % 1_000_000 == 0) System.out.println("wrote " + i);
            }
            stackCount.entrySet().stream().filter(e -> e.getValue() > 1).sorted(Comparator.comparingInt(Map.Entry::getValue)).forEach(System.out::println);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    tailerThread.setName("tail");
    tailerThread.start();
    appenderThread.setName("appd");
    appenderThread.start();
    appenderThread.join();
    // Pause to allow tailer to catch up (if needed)
    Jvm.pause(500);
    tailerThread.interrupt();
    tailerThread.join();
// System.out.println("wr: " + histogramWr.toMicrosFormat());
// System.out.println("in: " + histogramIn.toMicrosFormat());
// System.out.println("co: " + histogramCo.toMicrosFormat());
}

Example 3 with BytesStore

use of net.openhft.chronicle.bytes.BytesStore in project Chronicle-Queue by OpenHFT.

the class MicroToucher method execute.

public boolean execute() {
    final Wire bufferWire = appender.wire();
    if (bufferWire == null)
        return false;
    final long lastPosition = appender.lastPosition;
    final long lastPage = lastPosition & ~0xFFF;
    final long nextPage = (lastPosition + 0xFFF) & ~0xFFF;
    Bytes bytes = bufferWire.bytes();
    if (nextPage != lastPageTouched) {
        lastPageTouched = nextPage;
        try {
            // best effort
            final BytesStore bs = bytes.bytesStore();
            if (bs.inside(nextPage, 8))
                touchPage(nextPage, bs);
        } catch (Throwable ignored) {
        }
        return true;
    }
    lastPageToSync = lastPage;
    return false;
}

Example 4 with BytesStore

use of net.openhft.chronicle.bytes.BytesStore in project Chronicle-Queue by OpenHFT.

the class MicroToucher method bgExecute.

public void bgExecute() {
    final long lastPage = this.lastPageToSync;
    final long start = this.lastPageSynced;
    final long length = Math.min(8 << 20, lastPage - start);
    // System.out.println("len "+length);
    if (length < 8 << 20)
        return;
    final Wire bufferWire = appender.wire();
    if (bufferWire == null)
        return;
    BytesStore bytes = bufferWire.bytes().bytesStore();
    sync(bytes, start, length);
    this.lastPageSynced += length;
}

Example 5 with BytesStore

use of net.openhft.chronicle.bytes.BytesStore in project Chronicle-Queue by OpenHFT.

the class LatencyDistributionMain method runTest.

protected void runTest(@NotNull ChronicleQueue queue, @NotNull ChronicleQueue queue2) throws InterruptedException {
    Histogram histogramCo = new Histogram();
    Histogram histogramIn = new Histogram();
    Histogram histogramWr = new Histogram();
    Thread pretoucher = new Thread(() -> {
        ExcerptAppender appender = queue.acquireAppender();
        try {
            while (!Thread.currentThread().isInterrupted()) {
                appender.pretouch();
                Jvm.pause(50);
            }
        } catch (Exception e) {
            if (!appender.isClosed())
                e.printStackTrace();
        }
    });
    pretoucher.setDaemon(true);
    pretoucher.start();
    ExcerptAppender appender = queue.acquireAppender();
    // two queues as most like in a different process.
    ExcerptTailer tailer = queue2.createTailer();
    String name = getClass().getName();
    Thread tailerThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableTailerAffinity") || !Jvm.getBoolean("disableAffinity")) {
                lock = Affinity.acquireLock();
            }
            int counter = 0;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    // if (SAMPLING)
                    // sampler.thread(Thread.currentThread());
                    // boolean found = tailer.readDocument(myReadMarshallable);
                    boolean found;
                    try (DocumentContext dc = tailer.readingDocument()) {
                        found = dc.isPresent();
                        if (found) {
                            int count = counter++;
                            if (count == WARMUP) {
                                histogramCo.reset();
                                histogramIn.reset();
                                histogramWr.reset();
                            }
                            Bytes<?> bytes = dc.wire().bytes();
                            long startCo = bytes.readLong();
                            long startIn = bytes.readLong();
                            long now = System.nanoTime();
                            histogramCo.sample(now - startCo);
                            histogramIn.sample(now - startIn);
                            if (count % INTLOG_INTERVAL == 0)
                                System.out.println("read  " + count);
                        }
                    }
                /*
                        if (SAMPLING) {
                            StackTraceElement[] stack = sampler.getAndReset();
                            if (stack != null) {
                                if (!stack[0].getClassName().equals(name) &&
                                        !stack[0].getClassName().equals("java.lang.Thread")) {
                                    StringBuilder sb = new StringBuilder();
                                    Jvm.trimStackTrace(sb, stack);
                                   // System.out.println(sb);
                                }
                            } else if (!found) {
                                Thread.yield();
                            }
                        }
                        */
                } catch (Exception e) {
                    break;
                }
            }
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    Thread appenderThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Jvm.getBoolean("enableAppenderAffinity") || !Jvm.getBoolean("disableAffinity")) {
                lock = Affinity.acquireLock();
            }
            long next = System.nanoTime();
            long interval = 1_000_000_000 / throughput;
            Map<String, Integer> stackCount = new LinkedHashMap<>();
            BytesStore<?, ?> bytes24 = BytesStore.nativeStoreFrom(new byte[Main.size - 16]);
            for (int i = -WARMUP; i < iterations; i++) {
                long s0 = System.nanoTime();
                if (s0 < next) {
                    do ; while (System.nanoTime() < next);
                    // if we failed to come out of the spin loop on time, reset next.
                    next = System.nanoTime();
                }
                if (SAMPLING) {
                    sampler.thread(Thread.currentThread());
                }
                long start = System.nanoTime();
                try (@NotNull DocumentContext dc = appender.writingDocument(false)) {
                    Wire wire = dc.wire();
                    Bytes<?> bytes2 = wire.bytes();
                    // when it should have started
                    bytes2.writeLong(next);
                    // when it actually started.
                    bytes2.writeLong(start);
                    bytes2.write(bytes24);
                    ThroughputMain.addToEndOfCache(wire);
                }
                long time = System.nanoTime() - start;
                histogramWr.sample(start - next);
                if (SAMPLING && time > 1e3 && i > 0) {
                    StackTraceElement[] stack = sampler.getAndReset();
                    if (stack != null) {
                        if (!stack[0].getClassName().equals(name) && !stack[0].getClassName().equals("java.lang.Thread")) {
                            StringBuilder sb = new StringBuilder();
                            Jvm.trimStackTrace(sb, stack);
                            stackCount.compute(sb.toString(), (k, v) -> v == null ? 1 : v + 1);
                        }
                    }
                }
                next += interval;
                if (i % INTLOG_INTERVAL == 0)
                    System.out.println("wrote " + i);
            }
            stackCount.entrySet().stream().filter(e -> e.getValue() > 1).forEach(System.out::println);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    tailerThread.start();
    appenderThread.start();
    appenderThread.join();
    pretoucher.interrupt();
    pretoucher.join();
    // Pause to allow tailer to catch up (if needed)
    Jvm.pause(500);
    tailerThread.interrupt();
    tailerThread.join();
    System.out.println("wr: " + histogramWr.toLongMicrosFormat());
    System.out.println("in: " + histogramIn.toLongMicrosFormat());
    System.out.println("co: " + histogramCo.toLongMicrosFormat());
}