Examples with ChronicleQueue net.openhft.chronicle.queue.ChronicleQueue used on opensource projects

Example 46 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class QueueWriteJitterMain method run.

protected void run() {
    MappedFile.warmup();
    String path = "test-q-" + Time.uniqueId();
    System.out.println("Writing to " + path);
    Thread pretoucher = new Thread(() -> {
        try (ChronicleQueue q = createQueue(path)) {
            ExcerptAppender appender = q.acquireAppender();
            while (true) {
                Thread.sleep(50);
                appender.pretouch();
            }
        } catch (InterruptedException ie) {
            if (running)
                ie.printStackTrace();
        }
    });
    pretoucher.setDaemon(true);
    pretoucher.start();
    Thread writer = new Thread(() -> {
        try (ChronicleQueue q = createQueue(path)) {
            ExcerptAppender appender = q.acquireAppender();
            while (running) {
                writeStarted = System.nanoTime();
                Jvm.safepoint();
                try (DocumentContext dc = appender.writingDocument(false)) {
                    Jvm.safepoint();
                    Bytes<?> bytes = dc.wire().bytes();
                    for (int i = 0; i < size; i += 8) bytes.writeLong(i);
                    Jvm.safepoint();
                }
                Jvm.safepoint();
                writeStarted = Long.MAX_VALUE;
                waitForNext(Math.min(100, sampleTime));
            }
        }
    });
    writer.setDaemon(true);
    writer.start();
    // give it time to start
    Jvm.pause(100);
    try (ChronicleQueue q = createQueue(path)) {
        ExcerptTailer tailer = q.createTailer();
        long start0 = System.currentTimeMillis();
        do {
            if (writeStarted < Long.MAX_VALUE) {
                // overflow exists loop
                while (writeStarted + sampleTime * 1000 > System.nanoTime()) Thread.yield();
                if (writeStarted < Long.MAX_VALUE) {
                    StackTraceElement[] stes = writer.getStackTrace();
                    if (!stes[1].getMethodName().equals("waitForNext")) {
                        StringBuilder sb = new StringBuilder();
                        sb.append(PROFILE_OF_THE_THREAD);
                        Jvm.trimStackTrace(sb, stes);
                        System.out.println(sb);
                    }
                }
            }
            try (DocumentContext dc = tailer.readingDocument()) {
                if (!dc.isPresent())
                    waitForNext(Math.min(100, sampleTime));
            }
            Thread.yield();
        } while (System.currentTimeMillis() < start0 + runTime * 1_000);
    }
    running = false;
    pretoucher.interrupt();
    IOTools.deleteDirWithFiles(path, 2);
}

Example 47 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class NotCompleteTest method testMessageLeftNotComplete.

@Test
public void testMessageLeftNotComplete() {
    File tmpDir = DirectoryUtils.tempDir("testMessageLeftNotComplete");
    try (final ChronicleQueue queue = binary(tmpDir).testBlockSize().rollCycle(RollCycles.TEST_DAILY).build()) {
        ExcerptAppender appender = queue.acquireAppender().lazyIndexing(lazyIndexing);
        // start a message which was not completed.
        DocumentContext dc = appender.writingDocument();
        dc.wire().write("some").text("data");
    // didn't call dc.close();
    }
    final SingleChronicleQueue singleChronicleQueue = null;
    try (final ChronicleQueue queue = binary(tmpDir).testBlockSize().build()) {
        ExcerptTailer tailer = queue.createTailer();
        try (DocumentContext dc = tailer.readingDocument()) {
            assertFalse(dc.isPresent());
        }
        String expectedEager = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  wireType: !WireType BINARY_LIGHT,\n" + "  writePosition: [\n" + "    0,\n" + "    0\n" + "  ],\n" + "  roll: !SCQSRoll {\n" + "    length: !int 86400000,\n" + "    format: yyyyMMdd,\n" + "    epoch: 0\n" + "  },\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 442,\n" + "    lastIndex: 0\n" + "  },\n" + "  lastAcknowledgedIndexReplicated: -1,\n" + "  recovery: !TimedStoreRecovery {\n" + "    timeStamp: 0\n" + "  },\n" + "  deltaCheckpointInterval: 0,\n" + "  lastIndexReplicated: -1,\n" + "  sourceId: 0\n" + "}\n" + "# position: 442, header: -1\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 1\n" + "  544,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 544, header: -1\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 0\n" + "  0, 0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 640, header: -1 or 0\n" + "--- !!not-ready-data! #binary\n" + "...\n" + "# 130428 bytes remaining\n";
        String expectedLazy = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  wireType: !WireType BINARY_LIGHT,\n" + "  writePosition: [\n" + "    0,\n" + "    0\n" + "  ],\n" + "  roll: !SCQSRoll {\n" + "    length: !int 86400000,\n" + "    format: yyyyMMdd,\n" + "    epoch: 0\n" + "  },\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 0,\n" + "    lastIndex: 0\n" + "  },\n" + "  lastAcknowledgedIndexReplicated: -1,\n" + "  recovery: !TimedStoreRecovery {\n" + "    timeStamp: 0\n" + "  },\n" + "  deltaCheckpointInterval: 0,\n" + "  lastIndexReplicated: -1,\n" + "  sourceId: 0\n" + "}\n" + "# position: 442, header: -1 or 0\n" + "--- !!not-ready-data! #binary\n" + "...\n" + "# 130626 bytes remaining\n";
        assertEquals(lazyIndexing ? expectedLazy : expectedEager, queue.dump());
    }
    try (final ChronicleQueue queue = binary(tmpDir).testBlockSize().timeoutMS(500).build()) {
        ExcerptAppender appender = queue.acquireAppender();
        try (DocumentContext dc = appender.writingDocument()) {
            dc.wire().write("some").text("data");
        }
        String expectedEager = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  wireType: !WireType BINARY_LIGHT,\n" + "  writePosition: [\n" + "    33412,\n" + "    143503447293952\n" + "  ],\n" + "  roll: !SCQSRoll {\n" + "    length: !int 86400000,\n" + "    format: yyyyMMdd,\n" + "    epoch: 0\n" + "  },\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 442,\n" + "    lastIndex: 1\n" + "  },\n" + "  lastAcknowledgedIndexReplicated: -1,\n" + "  recovery: !TimedStoreRecovery {\n" + "    timeStamp: 0\n" + "  },\n" + "  deltaCheckpointInterval: 0,\n" + "  lastIndexReplicated: -1,\n" + "  sourceId: 0\n" + "}\n" + "# position: 442, header: -1\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 1\n" + "  544,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 544, header: -1\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 1\n" + "  33412,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 640, header: -1\n" + "--- !!meta-data #binary\n" + "\"!! Skipped due to recovery of locked header !!";
        String expectedEagerFooter = "# position: 33412, header: 0\n" + "--- !!data #binary\n" + "some: data\n" + "...\n" + "# 97642 bytes remaining\n";
        String expectedLazy = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  wireType: !WireType BINARY_LIGHT,\n" + "  writePosition: [\n" + "    33408,\n" + "    143486267424768\n" + "  ],\n" + "  roll: !SCQSRoll {\n" + "    length: !int 86400000,\n" + "    format: yyyyMMdd,\n" + "    epoch: 0\n" + "  },\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 33212,\n" + "    lastIndex: 1\n" + "  },\n" + "  lastAcknowledgedIndexReplicated: -1,\n" + "  recovery: !TimedStoreRecovery {\n" + "    timeStamp: 0\n" + "  },\n" + "  deltaCheckpointInterval: 0,\n" + "  lastIndexReplicated: -1,\n" + "  sourceId: 0\n" + "}\n" + "# position: 442, header: -1\n" + "--- !!meta-data #binary\n" + "\"!! Skipped due to recovery of locked header !!";
        String expectedLazyFooter = "# position: 33212, header: -1\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 1\n" + "  33312,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 33312, header: -1\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 1\n" + "  33408,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "# position: 33408, header: 0\n" + "--- !!data #binary\n" + "some: data\n" + "...\n" + "# 97646 bytes remaining\n";
        if (lazyIndexing) {
            assertThat(queue.dump(), containsString(expectedLazy));
            assertThat(queue.dump(), containsString(expectedLazyFooter));
        } else {
            assertThat(queue.dump(), containsString(expectedEager));
            assertThat(queue.dump(), containsString(expectedEagerFooter));
        }
    }
}

Example 48 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class NotCompleteTest method testInterruptedDuringSerialisation.

@Test
public void testInterruptedDuringSerialisation() throws InterruptedException {
    final File tmpDir = DirectoryUtils.tempDir("testInterruptedDuringSerialisation_" + (lazyIndexing ? "lazy" : "eager"));
    DirectoryUtils.deleteDir(tmpDir);
    tmpDir.mkdirs();
    final List<String> names = Collections.synchronizedList(new ArrayList<>());
    final Person person1 = new Person(40, "Terry");
    final Person interrupter = new Person(50, Person.INTERRUPT);
    final Person thrower = new Person(80, Person.THROW);
    final Person person2 = new Person(90, "Bert");
    try (final ChronicleQueue queueReader = binary(tmpDir).testBlockSize().rollCycle(RollCycles.TEST_DAILY).timeoutMS(500).build();
        final ChronicleQueue queueWriter = binary(tmpDir).testBlockSize().rollCycle(RollCycles.TEST_DAILY).build()) {
        ExcerptTailer tailer = queueReader.createTailer();
        MethodReader reader = tailer.methodReader((PersonListener) person -> names.add(person.name));
        final StringBuilder queueDumpBeforeInterruptedWrite = new StringBuilder();
        // set up
        doWrite(queueWriter, (proxy, queue) -> {
            proxy.accept(person1);
            queueDumpBeforeInterruptedWrite.append(queue.dump());
        });
        String cleanedQueueDump = cleanQueueDump(queueDumpBeforeInterruptedWrite.toString());
        // start up writer thread
        Thread writerThread = new Thread(() -> doWrite(queueWriter, (proxy, queue) -> {
            // thread is interrupted during this
            proxy.accept(interrupter);
        }));
        writerThread.start();
        writerThread.join();
        try (final ChronicleQueue queue = binary(tmpDir).testBlockSize().rollCycle(RollCycles.TEST_DAILY).build()) {
            String dump = cleanQueueDump(queue.dump());
            assertEquals("queue should be unchanged by the interrupted write", cleanedQueueDump, dump);
        }
        // check only 1 written
        assertTrue(reader.readOne());
        assertEquals(1, names.size());
        assertEquals(person1.name, names.get(0));
        assertFalse(reader.readOne());
        // do a write that throws an exception
        doWrite(queueWriter, (proxy, queue) -> {
            try {
                proxy.accept(thrower);
            } catch (NullPointerException npe) {
            // ignore
            }
        });
        try (final ChronicleQueue queue = binary(tmpDir).testBlockSize().rollCycle(RollCycles.TEST_DAILY).build()) {
            String dump = cleanQueueDump(queue.dump());
            if (lazyIndexing) {
                // reading the queue creates the index, thus changing it, so do a text comparison here
                cleanedQueueDump = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  wireType: !WireType BINARY_LIGHT,\n" + "  writePosition: [\n" + "    442,\n" + "    0\n" + "  ],\n" + "  roll: !SCQSRoll {\n" + "    length: !int 86400000,\n" + "    format: yyyyMMdd,\n" + "    epoch: 0\n" + "  },\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 475,\n" + "    lastIndex: 0\n" + "  },\n" + "  lastAcknowledgedIndexReplicated: -1,\n" + "  recovery: !TimedStoreRecovery {\n" + "    timeStamp: 0\n" + "  },\n" + "  deltaCheckpointInterval: 0,\n" + "  lastIndexReplicated: -1,\n" + "  sourceId: 0\n" + "}\n" + "# position: 442, header: 0\n" + "--- !!data #binary\n" + "accept: {\n" + "  age: 40,\n" + "  name: Terry\n" + "}\n" + "# position: 475, header: 0\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 0\n" + "  0, 0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "...\n" + "\n";
            }
            assertEquals("queue should be unchanged by the failed write", cleanedQueueDump, dump);
        }
        // check nothing else written
        assertFalse(reader.readOne());
        // do an empty write
        ExcerptAppender appender = queueWriter.acquireAppender().lazyIndexing(lazyIndexing);
        DocumentContext wd = appender.writingDocument();
        wd.rollbackOnClose();
        wd.close();
        // check queue unchanged
        String dump = cleanQueueDump(queueWriter.dump());
        assertEquals("queue should be unchanged by the failed write", cleanedQueueDump, dump);
        // check nothing else written
        assertFalse(reader.readOne());
        // write another person to same queue in this thread
        doWrite(queueWriter, (proxy, queue) -> proxy.accept(person2));
        assertTrue(reader.readOne());
        assertEquals(2, names.size());
        assertEquals(person2.name, names.get(1));
        assertFalse(reader.readOne());
    }
}

Example 49 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class SingleChronicleQueueCloseTest method testTailAfterClose.

@Test
public void testTailAfterClose() {
    final ChronicleQueue queue = SingleChronicleQueueBuilder.builder(getTmpDir(), WireType.BINARY).build();
    final ExcerptAppender appender = queue.acquireAppender();
    appender.writeDocument(w -> w.write(TestKey.test).int32(1));
    queue.close();
    try {
        appender.writeDocument(w -> w.write(TestKey.test).int32(2));
        Assert.fail();
    } catch (IllegalStateException e) {
    // ok
    }
}

Example 50 with ChronicleQueue

use of net.openhft.chronicle.queue.ChronicleQueue in project Chronicle-Queue by OpenHFT.

the class Queue30Test method testMT.

@Ignore("Stress test - doesn't finish")
@Test
public void testMT() throws IOException, InterruptedException {
    try (final ChronicleQueue queue = SingleChronicleQueueBuilder.text(getTmpDir()).blockSize(640_000).build()) {
        ExecutorService exec = Executors.newCachedThreadPool(new NamedThreadFactory("stress"));
        Throwable[] tref = { null };
        Runnable r = () -> {
            try {
                final String name = Thread.currentThread().getName();
                final ExcerptAppender appender = queue.acquireAppender();
                for (int count = 0; !Thread.currentThread().isInterrupted(); count++) {
                    final int c = count;
                    appender.writeDocument(w -> w.write(() -> "thread").text(name).write(() -> "count").int32(c));
                    if (count % 10_000 == 0) {
                        LOGGER.info(name + "> " + count);
                    }
                }
            } catch (Throwable t) {
                tref[0] = t;
                exec.shutdown();
            }
        };
        for (int i = 0; i < 100; i++) exec.submit(r);
        exec.awaitTermination(10, TimeUnit.MINUTES);
        exec.shutdownNow();
        if (tref[0] != null)
            throw new AssertionError(tref[0]);
    }
}