Examples with RollingChronicleQueue net.openhft.chronicle.queue.impl.RollingChronicleQueue used on opensource projects

Example 1 with RollingChronicleQueue

use of net.openhft.chronicle.queue.impl.RollingChronicleQueue in project Chronicle-Queue by OpenHFT.

the class SingleCQFormat2Test method testWritingTwentyMessagesTinyIndex.

@Test
public void testWritingTwentyMessagesTinyIndex() throws FileNotFoundException {
    for (int spacing : new int[] { 1, 2, 4 }) {
        @NotNull File dir = getTmpDir();
        dir.mkdir();
        try (@NotNull RollingChronicleQueue queue = binary(dir).blockSize(QueueUtil.testBlockSize()).indexCount(8).indexSpacing(spacing).build()) {
            long start = RollCycles.DEFAULT.toIndex(queue.cycle(), 0);
            @NotNull ExcerptTailer tailer = queue.createTailer();
            assertFalse(tailer.moveToIndex(start));
            appendMessage(queue, start, "Hello World");
            @NotNull String expectedEager = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  writePosition: [\n" + "    392,\n" + "    1683627180032\n" + "  ],\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 196,\n" + "    lastIndex: 1\n" + "  },\n" + "  dataFormat: 1\n" + "}\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 1\n" + "  296,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 1\n" + "  392,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "--- !!data #binary\n" + "msg: Hello World\n" + "...\n";
            checkFileContents(getFirstQueueFile(dir), expectedEager.replace("indexSpacing: 1", "indexSpacing: " + spacing).replace("lastIndex: 1", "lastIndex: " + spacing));
            assertTrue(tailer.moveToIndex(start));
            for (int i = 1; i < 19; i++) {
                assertFalse(tailer.moveToIndex(start + i));
                appendMessage(queue, start + i, "Another Hello World " + (i + 1));
                assertTrue(tailer.moveToIndex(start + i));
            }
            assertFalse(tailer.moveToIndex(start + 19));
            appendMessage(queue, start + 19, "Bye for now");
            assertTrue(tailer.moveToIndex(start + 19));
            assertFalse(tailer.moveToIndex(start + 20));
            @NotNull String expected1 = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  writePosition: [\n" + "    1176,\n" + "    5050881540115\n" + "  ],\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 1,\n" + "    index2Index: 196,\n" + "    lastIndex: 20\n" + "  },\n" + "  dataFormat: 1\n" + "}\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 3\n" + "  296,\n" + "  668,\n" + "  1016,\n" + "  0, 0, 0, 0, 0\n" + "]\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 8\n" + "  392,\n" + "  412,\n" + "  444,\n" + "  476,\n" + "  508,\n" + "  540,\n" + "  572,\n" + "  604\n" + "]\n" + "--- !!data #binary\n" + "msg: Hello World\n" + "--- !!data #binary\n" + "msg: Another Hello World 2\n" + "--- !!data #binary\n" + "msg: Another Hello World 3\n" + "--- !!data #binary\n" + "msg: Another Hello World 4\n" + "--- !!data #binary\n" + "msg: Another Hello World 5\n" + "--- !!data #binary\n" + "msg: Another Hello World 6\n" + "--- !!data #binary\n" + "msg: Another Hello World 7\n" + "--- !!data #binary\n" + "msg: Another Hello World 8\n" + "--- !!data #binary\n" + "msg: Another Hello World 9\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 8\n" + "  636,\n" + "  760,\n" + "  792,\n" + "  824,\n" + "  856,\n" + "  888,\n" + "  920,\n" + "  952\n" + "]\n" + "--- !!data #binary\n" + "msg: Another Hello World 10\n" + "--- !!data #binary\n" + "msg: Another Hello World 11\n" + "--- !!data #binary\n" + "msg: Another Hello World 12\n" + "--- !!data #binary\n" + "msg: Another Hello World 13\n" + "--- !!data #binary\n" + "msg: Another Hello World 14\n" + "--- !!data #binary\n" + "msg: Another Hello World 15\n" + "--- !!data #binary\n" + "msg: Another Hello World 16\n" + "--- !!data #binary\n" + "msg: Another Hello World 17\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 4\n" + "  984,\n" + "  1112,\n" + "  1144,\n" + "  1176,\n" + "  0, 0, 0, 0\n" + "]\n" + "--- !!data #binary\n" + "msg: Another Hello World 18\n" + "--- !!data #binary\n" + "msg: Another Hello World 19\n" + "--- !!data #binary\n" + "msg: Bye for now\n" + "...\n";
            @NotNull String expected2 = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  writePosition: [\n" + "    1080,\n" + "    4638564679699\n" + "  ],\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 2,\n" + "    index2Index: 196,\n" + "    lastIndex: 20\n" + "  },\n" + "  dataFormat: 1\n" + "}\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 2\n" + "  296,\n" + "  924,\n" + "  0, 0, 0, 0, 0, 0\n" + "]\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 8\n" + "  392,\n" + "  444,\n" + "  508,\n" + "  572,\n" + "  636,\n" + "  700,\n" + "  764,\n" + "  828\n" + "]\n" + "--- !!data #binary\n" + "msg: Hello World\n" + "--- !!data #binary\n" + "msg: Another Hello World 2\n" + "--- !!data #binary\n" + "msg: Another Hello World 3\n" + "--- !!data #binary\n" + "msg: Another Hello World 4\n" + "--- !!data #binary\n" + "msg: Another Hello World 5\n" + "--- !!data #binary\n" + "msg: Another Hello World 6\n" + "--- !!data #binary\n" + "msg: Another Hello World 7\n" + "--- !!data #binary\n" + "msg: Another Hello World 8\n" + "--- !!data #binary\n" + "msg: Another Hello World 9\n" + "--- !!data #binary\n" + "msg: Another Hello World 10\n" + "--- !!data #binary\n" + "msg: Another Hello World 11\n" + "--- !!data #binary\n" + "msg: Another Hello World 12\n" + "--- !!data #binary\n" + "msg: Another Hello World 13\n" + "--- !!data #binary\n" + "msg: Another Hello World 14\n" + "--- !!data #binary\n" + "msg: Another Hello World 15\n" + "--- !!data #binary\n" + "msg: Another Hello World 16\n" + "--- !!data #binary\n" + "msg: Another Hello World 17\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 2\n" + "  892,\n" + "  1048,\n" + "  0, 0, 0, 0, 0, 0\n" + "]\n" + "--- !!data #binary\n" + "msg: Another Hello World 18\n" + "--- !!data #binary\n" + "msg: Another Hello World 19\n" + "--- !!data #binary\n" + "msg: Bye for now\n" + "...\n";
            @NotNull String expected3 = "--- !!meta-data #binary\n" + "header: !SCQStore {\n" + "  writePosition: [\n" + "    988,\n" + "    4243427688467\n" + "  ],\n" + "  indexing: !SCQSIndexing {\n" + "    indexCount: 8,\n" + "    indexSpacing: 4,\n" + "    index2Index: 196,\n" + "    lastIndex: 20\n" + "  },\n" + "  dataFormat: 1\n" + "}\n" + "--- !!meta-data #binary\n" + "index2index: [\n" + "  # length: 8, used: 1\n" + "  296,\n" + "  0, 0, 0, 0, 0, 0, 0\n" + "]\n" + "--- !!meta-data #binary\n" + "index: [\n" + "  # length: 8, used: 5\n" + "  392,\n" + "  508,\n" + "  636,\n" + "  764,\n" + "  892,\n" + "  0, 0, 0\n" + "]\n" + "--- !!data #binary\n" + "msg: Hello World\n" + "--- !!data #binary\n" + "msg: Another Hello World 2\n" + "--- !!data #binary\n" + "msg: Another Hello World 3\n" + "--- !!data #binary\n" + "msg: Another Hello World 4\n" + "--- !!data #binary\n" + "msg: Another Hello World 5\n" + "--- !!data #binary\n" + "msg: Another Hello World 6\n" + "--- !!data #binary\n" + "msg: Another Hello World 7\n" + "--- !!data #binary\n" + "msg: Another Hello World 8\n" + "--- !!data #binary\n" + "msg: Another Hello World 9\n" + "--- !!data #binary\n" + "msg: Another Hello World 10\n" + "--- !!data #binary\n" + "msg: Another Hello World 11\n" + "--- !!data #binary\n" + "msg: Another Hello World 12\n" + "--- !!data #binary\n" + "msg: Another Hello World 13\n" + "--- !!data #binary\n" + "msg: Another Hello World 14\n" + "--- !!data #binary\n" + "msg: Another Hello World 15\n" + "--- !!data #binary\n" + "msg: Another Hello World 16\n" + "--- !!data #binary\n" + "msg: Another Hello World 17\n" + "--- !!data #binary\n" + "msg: Another Hello World 18\n" + "--- !!data #binary\n" + "msg: Another Hello World 19\n" + "--- !!data #binary\n" + "msg: Bye for now\n" + "...\n";
            @NotNull String expected = spacing == 1 ? expected1 : spacing == 2 ? expected2 : expected3;
            checkFileContents(getFirstQueueFile(dir), expected);
        }
    }
}

Example 2 with RollingChronicleQueue

use of net.openhft.chronicle.queue.impl.RollingChronicleQueue in project Chronicle-Queue by OpenHFT.

the class EofMarkerOnEmptyQueueTest method shouldRecoverFromEmptyQueueOnRoll.

@Test
public void shouldRecoverFromEmptyQueueOnRoll() throws IOException, InterruptedException, ExecutionException, TimeoutException {
    Assume.assumeFalse(OS.isWindows());
    expectException("Couldn't acquire write lock");
    expectException("Forced unlock for the lock");
    final AtomicLong clock = new AtomicLong(System.currentTimeMillis());
    try (final RollingChronicleQueue queue = ChronicleQueue.singleBuilder(tmpFolder.newFolder()).rollCycle(RollCycles.TEST_SECONDLY).timeProvider(clock::get).timeoutMS(1_000).testBlockSize().build()) {
        final ExcerptAppender appender = queue.acquireAppender();
        final DocumentContext context = appender.writingDocument();
        // start to write a message, but don't close the context - simulates crashed writer
        final long expectedEofMarkerPosition = context.wire().bytes().writePosition() - Wires.SPB_HEADER_SIZE;
        context.wire().write("foo").int32(1);
        final int startCycle = queue.cycle();
        clock.addAndGet(TimeUnit.SECONDS.toMillis(1L));
        final int nextCycle = queue.cycle();
        // ensure that the cycle file will roll
        assertNotEquals(nextCycle, startCycle);
        ExecutorService appenderExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory("Appender"));
        appenderExecutor.submit(() -> {
            try (final DocumentContext nextCtx = queue.acquireAppender().writingDocument()) {
                nextCtx.wire().write("bar").int32(7);
            }
        }).get(Jvm.isDebug() ? 3000 : 3, TimeUnit.SECONDS);
        appenderExecutor.shutdown();
        appenderExecutor.awaitTermination(1, TimeUnit.SECONDS);
        try (final SingleChronicleQueueStore firstCycleStore = queue.storeForCycle(startCycle, 0, false, null)) {
            final long firstCycleWritePosition = firstCycleStore.writePosition();
            // assert that no write was completed
            assertEquals(0L, firstCycleWritePosition);
            // firstCycleStore.release(test);
            final ExcerptTailer tailer = queue.createTailer();
            int recordCount = 0;
            int lastItem = -1;
            while (true) {
                try (final DocumentContext readCtx = tailer.readingDocument()) {
                    if (!readCtx.isPresent()) {
                        break;
                    }
                    final StringBuilder name = new StringBuilder();
                    final ValueIn field = readCtx.wire().readEventName(name);
                    recordCount++;
                    lastItem = field.int32();
                }
            }
            assertEquals(1, recordCount);
            assertEquals(7, lastItem);
        }
    }
}

Example 3 with RollingChronicleQueue

use of net.openhft.chronicle.queue.impl.RollingChronicleQueue in project Chronicle-Queue by OpenHFT.

the class QueueLockTest method check.

private void check(boolean shouldThrowException) throws InterruptedException {
    finishedNormally = false;
    ignoreException("Couldn't acquire write lock");
    if (!shouldThrowException)
        expectException("Forced unlock for the lock");
    try {
        System.setProperty("queue.dont.recover.lock.timeout", Boolean.toString(shouldThrowException));
        final long timeoutMs = 2_000;
        final File queueDir = DirectoryUtils.tempDir("check");
        try (final RollingChronicleQueue queue = ChronicleQueue.singleBuilder(queueDir).timeoutMS(timeoutMs).build()) {
            // lock the queue
            try (DocumentContext dc = queue.acquireAppender().writingDocument()) {
                final CountDownLatch started = new CountDownLatch(1);
                final CountDownLatch finished = new CountDownLatch(1);
                final AtomicBoolean recoveredAndAcquiredTheLock = new AtomicBoolean();
                final AtomicBoolean threwException = new AtomicBoolean();
                final Thread otherWriter = new Thread(() -> {
                    try (final RollingChronicleQueue queue2 = ChronicleQueue.singleBuilder(queueDir).timeoutMS(timeoutMs).build()) {
                        started.countDown();
                        try (DocumentContext ignored = queue2.acquireAppender().writingDocument()) {
                            recoveredAndAcquiredTheLock.set(true);
                            System.out.println("Done");
                        } catch (UnrecoverableTimeoutException e) {
                            e.printStackTrace();
                            threwException.set(true);
                        } catch (Throwable t) {
                            t.printStackTrace();
                        } finally {
                            System.out.println("finished");
                            finished.countDown();
                        }
                    }
                }, "Test thread");
                otherWriter.start();
                long startTime = System.currentTimeMillis();
                started.await(1, TimeUnit.SECONDS);
                finished.await(10, TimeUnit.SECONDS);
                long endTime = System.currentTimeMillis();
                long time = endTime - startTime;
                assertEquals(shouldThrowException, threwException.get());
                assertEquals(shouldThrowException, !recoveredAndAcquiredTheLock.get());
                assertTrue("timeout, time: " + time, time >= timeoutMs);
            }
        }
        finishedNormally = true;
    } finally {
        System.clearProperty("queue.dont.recover.lock.timeout");
    }
}

Example 4 with RollingChronicleQueue

use of net.openhft.chronicle.queue.impl.RollingChronicleQueue in project Chronicle-Queue by OpenHFT.

the class SingleCQFormatTest method testEmptyDirectory.

@Test
public void testEmptyDirectory() {
    final File dir = new File(OS.getTarget(), getClass().getSimpleName() + "-" + Time.uniqueId());
    dir.mkdir();
    try (RollingChronicleQueue queue = binary(dir).testBlockSize().build()) {
        assertEquals(Integer.MAX_VALUE, queue.firstCycle());
        assertEquals(Long.MAX_VALUE, queue.firstIndex());
        assertEquals(Integer.MIN_VALUE, queue.lastCycle());
    }
    IOTools.shallowDeleteDirWithFiles(dir.getAbsolutePath());
}

Example 5 with RollingChronicleQueue

use of net.openhft.chronicle.queue.impl.RollingChronicleQueue in project Chronicle-Queue by OpenHFT.

the class SingleCQFormatTest method testInvalidFile.

@Test
public void testInvalidFile() throws FileNotFoundException {
    // based on the file name
    expectException("Overriding roll cycle from TEST4_DAILY to DAILY");
    final File dir = new File(OS.getTarget() + "/deleteme-" + Time.uniqueId());
    dir.mkdir();
    try (MappedBytes bytes = MappedBytes.mappedBytes(new File(dir, "19700102" + SingleChronicleQueue.SUFFIX), 64 << 10)) {
        bytes.write8bit("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\" ?>");
        try (RollingChronicleQueue queue = binary(dir).rollCycle(RollCycles.TEST4_DAILY).testBlockSize().build()) {
            assertEquals(1, queue.firstCycle());
            assertEquals(1, queue.lastCycle());
            try {
                final ExcerptTailer tailer = queue.createTailer();
                tailer.toEnd();
                fail();
            } catch (Exception e) {
                assertEquals("java.io.StreamCorruptedException: Unexpected magic number 783f3c37", e.toString());
            }
        }
    }
    System.gc();
    try {
        IOTools.shallowDeleteDirWithFiles(dir.getAbsolutePath());
    } catch (Exception e) {
        e.printStackTrace();
    }
}