Examples with NativeBytesStore net.openhft.chronicle.bytes.NativeBytesStore used on opensource projects

Example 1 with NativeBytesStore

use of net.openhft.chronicle.bytes.NativeBytesStore in project Chronicle-Queue by OpenHFT.

the class ChronicleQueueLatencyDistribution method runTest.

protected void runTest(@NotNull ChronicleQueue queue, int throughput) throws InterruptedException {
    /*
        Jvm.setExceptionHandlers(PrintExceptionHandler.ERR,
                PrintExceptionHandler.OUT,
                PrintExceptionHandler.OUT);
*/
    Histogram histogramCo = new Histogram();
    Histogram histogramIn = new Histogram();
    Histogram histogramWr = new Histogram();
    Thread pretoucher = new Thread(() -> {
        ExcerptAppender appender = queue.acquireAppender();
        while (!Thread.currentThread().isInterrupted()) {
            appender.pretouch();
            Jvm.pause(500);
        }
    });
    pretoucher.setDaemon(true);
    pretoucher.start();
    ExcerptAppender appender = queue.acquireAppender().lazyIndexing(true);
    ExcerptTailer tailer = queue.createTailer();
    String name = getClass().getName();
    Thread tailerThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Boolean.getBoolean("enableTailerAffinity") || Boolean.getBoolean("enableAffinity")) {
                lock = Affinity.acquireLock();
            }
            int counter = 0;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    // if (SAMPLING)
                    // sampler.thread(Thread.currentThread());
                    // boolean found = tailer.readDocument(myReadMarshallable);
                    boolean found;
                    try (DocumentContext dc = tailer.readingDocument()) {
                        found = dc.isPresent();
                        if (found) {
                            int count = counter++;
                            if (count == warmup) {
                                histogramCo.reset();
                                histogramIn.reset();
                                histogramWr.reset();
                            }
                            Bytes<?> bytes = dc.wire().bytes();
                            long startCo = bytes.readLong();
                            long startIn = bytes.readLong();
                            long now = System.nanoTime();
                            histogramCo.sample(now - startCo);
                            histogramIn.sample(now - startIn);
                        }
                    }
                /*
                        if (SAMPLING) {
                            StackTraceElement[] stack = sampler.getAndReset();
                            if (stack != null) {
                                if (!stack[0].getClassName().equals(name) &&
                                        !stack[0].getClassName().equals("java.lang.Thread")) {
                                    StringBuilder sb = new StringBuilder();
                                    Jvm.trimStackTrace(sb, stack);
                                    System.out.println(sb);
                                }
                            } else if (!found) {
                                Thread.yield();
                            }
                        }
                        */
                } catch (Exception e) {
                    break;
                }
            }
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    Thread appenderThread = new Thread(() -> {
        AffinityLock lock = null;
        try {
            if (Boolean.getBoolean("enableAppenderAffinity") || Boolean.getBoolean("enableAffinity")) {
                lock = Affinity.acquireLock();
            }
            long next = System.nanoTime();
            long interval = 1_000_000_000 / throughput;
            Map<String, Integer> stackCount = new LinkedHashMap<>();
            NativeBytesStore bytes24 = NativeBytesStore.from(new byte[24]);
            for (int i = -warmup; i < 20_000_000; i++) {
                long s0 = System.nanoTime();
                if (s0 < next) {
                    busyLoop: do ; while (System.nanoTime() < next);
                    // if we failed to come out of the spin loop on time, reset next.
                    next = System.nanoTime();
                }
                if (SAMPLING) {
                    sampler.thread(Thread.currentThread());
                }
                long start = System.nanoTime();
                try (@NotNull DocumentContext dc = appender.writingDocument(false)) {
                    Bytes<?> bytes2 = dc.wire().bytes();
                    // when it should have started
                    bytes2.writeLong(next);
                    // when it actually started.
                    bytes2.writeLong(start);
                    bytes2.write(bytes24);
                }
                long time = System.nanoTime() - start;
                histogramWr.sample(start - next);
                if (SAMPLING && time > 1e3 && i > 0) {
                    StackTraceElement[] stack = sampler.getAndReset();
                    if (stack != null) {
                        if (!stack[0].getClassName().equals(name) && !stack[0].getClassName().equals("java.lang.Thread")) {
                            StringBuilder sb = new StringBuilder();
                            Jvm.trimStackTrace(sb, stack);
                            stackCount.compute(sb.toString(), (k, v) -> v == null ? 1 : v + 1);
                        }
                    }
                }
                next += interval;
            }
            stackCount.entrySet().stream().filter(e -> e.getValue() > 1).forEach(System.out::println);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (lock != null) {
                lock.release();
            }
        }
    });
    tailerThread.start();
    appenderThread.start();
    appenderThread.join();
    // Pause to allow tailer to catch up (if needed)
    Jvm.pause(500);
    tailerThread.interrupt();
    tailerThread.join();
    System.out.println("wr: " + histogramWr.toMicrosFormat());
    System.out.println("in: " + histogramIn.toMicrosFormat());
    System.out.println("co: " + histogramCo.toMicrosFormat());
}

Example 2 with NativeBytesStore

use of net.openhft.chronicle.bytes.NativeBytesStore in project Chronicle-Queue by OpenHFT.

the class BytesRingBufferTest method testMultiThreadedWithIntValues.

// @Ignore("works in lang-bytes, appears to be a visibility issue that can be fixed by adding
// a" +
// " synchronized to ringbuffer.poll() and ringbuffer.offer()")
@Test
public void testMultiThreadedWithIntValues() {
    try (NativeBytesStore allocate = NativeBytesStore.nativeStoreWithFixedCapacity(1000)) {
        final BytesRingBuffer bytesRingBuffer = new BytesRingBuffer(allocate.bytes());
        AtomicInteger counter = new AtomicInteger();
        // writer
        int iterations = 20_000;
        {
            ExecutorService executorService = Executors.newFixedThreadPool(2);
            for (int i = 0; i < iterations; i++) {
                final int j = i;
                executorService.submit(() -> {
                    try (NativeBytesStore allocate2 = NativeBytesStore.nativeStoreWithFixedCapacity(iterations)) {
                        final Bytes out = allocate2.bytes();
                        out.clear();
                        out.writeInt(j);
                        counter.addAndGet(j);
                        out.flip();
                        boolean offer;
                        do {
                            offer = bytesRingBuffer.offer(out);
                        } while (!offer);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                });
            }
        }
        CountDownLatch count = new CountDownLatch(iterations);
        // reader
        {
            ExecutorService executorService = Executors.newSingleThreadExecutor();
            for (int i = 0; i < iterations; i++) {
                executorService.submit(() -> {
                    try {
                        try (NativeBytesStore allocate3 = NativeBytesStore.nativeStoreWithFixedCapacity(25)) {
                            final Bytes bytes = allocate3.bytes();
                            Bytes result = null;
                            do {
                                try {
                                    result = bytesRingBuffer.poll(maxsize -> bytes);
                                } catch (InterruptedException e) {
                                    return;
                                }
                            } while (result == null);
                            int value = result.readInt();
                            counter.addAndGet(-value);
                            count.countDown();
                        } catch (Error e) {
                            e.printStackTrace();
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                });
            }
        }
        Assert.assertTrue(count.await(5000, TimeUnit.SECONDS));
        Assert.assertEquals(0, counter.get());
    }
}

Example 3 with NativeBytesStore

use of net.openhft.chronicle.bytes.NativeBytesStore in project Chronicle-Queue by OpenHFT.

the class BytesRingBufferTest method testMultiThreadedCheckAllEntriesReturnedAreValidText.

// @Ignore("works in lang-bytes")
@Test
public void testMultiThreadedCheckAllEntriesReturnedAreValidText() throws InterruptedException {
    System.out.println("Hello World");
    try (NativeBytesStore allocate = NativeBytesStore.nativeStoreWithFixedCapacity(1000)) {
        final BytesRingBuffer bytesRingBuffer = new BytesRingBuffer(allocate.bytes());
        // writer
        final int iterations = 100_000;
        {
            ExecutorService executorService = Executors.newFixedThreadPool(2);
            for (int i = 0; i < iterations; i++) {
                final int j = i;
                executorService.submit(() -> {
                    try (NativeBytesStore<Void> nativeStore = NativeBytesStore.nativeStoreWithFixedCapacity(iterations)) {
                        final Bytes out = nativeStore.bytes();
                        String expected = EXPECTED_VALUE + j;
                        out.clear();
                        out.writeUTFΔ(expected);
                        out.flip();
                        boolean offer;
                        do {
                            offer = bytesRingBuffer.offer(out);
                        } while (!offer);
                        System.out.println("+");
                    } catch (Throwable e) {
                        e.printStackTrace();
                    }
                });
            }
        }
        CountDownLatch count = new CountDownLatch(iterations);
        System.out.println(count);
        // reader
        {
            ExecutorService executorService = Executors.newSingleThreadExecutor();
            for (int i = 0; i < iterations; i++) {
                executorService.submit(() -> {
                    try (NativeBytesStore<Void> nativeStore = NativeBytesStore.nativeStoreWithFixedCapacity(25)) {
                        Bytes bytes = nativeStore.bytes();
                        Bytes result = null;
                        do {
                            try {
                                result = bytesRingBuffer.poll(maxSize -> bytes);
                            } catch (InterruptedException e) {
                                return;
                            }
                        } while (result == null);
                        result.clear();
                        String actual = result.readUTFΔ();
                        if (actual.startsWith(EXPECTED_VALUE))
                            count.countDown();
                        System.out.println("-");
                    } catch (Throwable e) {
                        e.printStackTrace();
                    }
                });
            }
        }
        Assert.assertTrue(count.await(5000, TimeUnit.SECONDS));
    }
}

Example 4 with NativeBytesStore

use of net.openhft.chronicle.bytes.NativeBytesStore in project Chronicle-Queue by OpenHFT.

the class DirectChronicleQueueStringTest method writeSome.

private void writeSome(DirectChronicleQueue chronicle) {
    NativeBytesStore allocate = NativeBytesStore.nativeStoreWithFixedCapacity(EXPECTED_BYTES.length);
    final Bytes toWrite = allocate.bytes();
    for (int i = 0; i < RUNS; i++) {
        toWrite.clear();
        toWrite.write(EXPECTED_BYTES);
        toWrite.flip();
        chronicle.appendDocument(toWrite);
    }
}

Example 5 with NativeBytesStore

use of net.openhft.chronicle.bytes.NativeBytesStore in project Chronicle-Queue by OpenHFT.

the class ZipBytesRingBufferTest method testZipAndAppend.

@Test
public void testZipAndAppend() {
    File file = null;
    try {
        NativeBytesStore allocate = NativeBytesStore.nativeStoreWithFixedCapacity(1024);
        NativeBytesStore msgBytes = NativeBytesStore.nativeStoreWithFixedCapacity(150);
        net.openhft.chronicle.bytes.Bytes message = msgBytes.bytes();
        message.writeUTFΔ("Hello World");
        message.flip();
        file = File.createTempFile("chronicle", "q");
        DirectChronicleQueue chronicle = (DirectChronicleQueue) new ChronicleQueueBuilder(file.getName()).build();
        final long writeAddress = getHeader((SingleChronicleQueue) chronicle).getWriteByte();
        final BytesRingBuffer ring = new BytesRingBuffer(allocate.bytes());
        final ZippedDocumentAppender zippedDocumentAppender = new ZippedDocumentAppender(ring, chronicle);
        zippedDocumentAppender.append(message);
        long initialValue = chronicle.firstBytes();
        AtomicLong offset = new AtomicLong(initialValue);
        while (lastWrite((SingleChronicleQueue) chronicle) == writeAddress) {
        // wait for data to be written ( via another thread )
        }
        // read the data from chronicle into actual
        Bytes actual = NativeBytesStore.nativeStoreWithFixedCapacity(100).bytes();
        chronicle.readDocument(offset, actual);
        // "Hello World" zipped should be 12 chars
        Assert.assertEquals(12, actual.flip().remaining());
    } finally {
        if (file != null)
            file.delete();
    }
}