Examples with PagedReference org.apache.activemq.artemis.core.paging.cursor.PagedReference used on opensource projects

Example 1 with PagedReference

use of org.apache.activemq.artemis.core.paging.cursor.PagedReference in project activemq-artemis by apache.

the class PageCursorStressTest method testSimpleCursor.

@Test
public void testSimpleCursor() throws Exception {
    final int NUM_MESSAGES = 100;
    PageSubscription cursor = lookupPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    Iterator<PagedReference> iterEmpty = cursor.iterator();
    int numberOfPages = addMessages(NUM_MESSAGES, 1024 * 1024);
    System.out.println("NumberOfPages = " + numberOfPages);
    PagedReference msg;
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    int key = 0;
    while ((msg = iterator.next()) != null) {
        assertEquals(key++, msg.getMessage().getIntProperty("key").intValue());
        cursor.confirmPosition(msg.getPosition());
    }
    assertEquals(NUM_MESSAGES, key);
    server.getStorageManager().waitOnOperations();
    waitCleanup();
    assertFalse(lookupPageStore(ADDRESS).isPaging());
    assertEquals(1, lookupPageStore(ADDRESS).getNumberOfPages());
    forceGC();
    server.stop();
    createServer();
    waitCleanup();
    assertEquals(1, lookupPageStore(ADDRESS).getNumberOfPages());
}

Example 2 with PagedReference

use of org.apache.activemq.artemis.core.paging.cursor.PagedReference in project activemq-artemis by apache.

the class PageCursorStressTest method testLazyCommit.

@Test
public void testLazyCommit() throws Exception {
    PagingStoreImpl pageStore = lookupPageStore(ADDRESS);
    pageStore.startPaging();
    final int NUM_MESSAGES = 100;
    final int messageSize = 100 * 1024;
    PageCursorProvider cursorProvider = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider();
    System.out.println("cursorProvider = " + cursorProvider);
    PageSubscription cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    System.out.println("Cursor: " + cursor);
    StorageManager storage = this.server.getStorageManager();
    long pgtxLazy = storage.generateID();
    Transaction txLazy = pgMessages(storage, pageStore, pgtxLazy, 0, NUM_MESSAGES, messageSize);
    addMessages(100, NUM_MESSAGES, messageSize);
    System.out.println("Number of pages - " + pageStore.getNumberOfPages());
    // First consume what's already there without any tx as nothing was committed
    for (int i = 100; i < 200; i++) {
        PagedReference pos = iterator.next();
        assertNotNull("Null at position " + i, pos);
        assertEquals(i, pos.getMessage().getIntProperty("key").intValue());
        cursor.ack(pos);
    }
    assertNull(iterator.next());
    txLazy.commit();
    storage.waitOnOperations();
    for (int i = 0; i < 100; i++) {
        PagedReference pos = iterator.next();
        assertNotNull("Null at position " + i, pos);
        assertEquals(i, pos.getMessage().getIntProperty("key").intValue());
        cursor.ack(pos);
    }
    assertNull(iterator.next());
    waitCleanup();
    server.stop();
    createServer();
    waitCleanup();
    assertEquals(1, lookupPageStore(ADDRESS).getNumberOfPages());
}

Example 3 with PagedReference

use of org.apache.activemq.artemis.core.paging.cursor.PagedReference in project activemq-artemis by apache.

the class PageCursorStressTest method testConsumeLivePage.

@Test
public void testConsumeLivePage() throws Exception {
    PagingStoreImpl pageStore = lookupPageStore(ADDRESS);
    pageStore.startPaging();
    final int NUM_MESSAGES = 100;
    final int messageSize = 1024 * 1024;
    PageCursorProvider cursorProvider = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider();
    System.out.println("cursorProvider = " + cursorProvider);
    PageSubscription cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    System.out.println("Cursor: " + cursor);
    RoutingContextImpl ctx = generateCTX();
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES; i++) {
        // if (i % 100 == 0)
        System.out.println("read/written " + i);
        ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
        Message msg = new CoreMessage(i, buffer.writerIndex());
        msg.putIntProperty("key", i);
        msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
        Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
        assertNull(iterator.next());
    }
    OperationContextImpl.clearContext();
    ctx = generateCTX();
    pageStore = lookupPageStore(ADDRESS);
    cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES * 2; i++) {
        if (i % 100 == 0)
            System.out.println("Paged " + i);
        if (i >= NUM_MESSAGES) {
            ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
            Message msg = new CoreMessage(i, buffer.writerIndex());
            msg.putIntProperty("key", i);
            msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
            Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        }
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
    }
    OperationContextImpl.clearContext();
    pageStore = lookupPageStore(ADDRESS);
    cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES * 3; i++) {
        if (i % 100 == 0)
            System.out.println("Paged " + i);
        if (i >= NUM_MESSAGES * 2 - 1) {
            ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
            Message msg = new CoreMessage(i, buffer.writerIndex());
            msg.putIntProperty("key", i + 1);
            msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
            Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        }
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        cursor.ack(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
    }
    PagedReference readMessage = iterator.next();
    assertEquals(NUM_MESSAGES * 3, readMessage.getMessage().getIntProperty("key").intValue());
    cursor.ack(readMessage);
    server.getStorageManager().waitOnOperations();
    pageStore.flushExecutors();
    assertFalse(pageStore.isPaging());
    server.stop();
    createServer();
    assertFalse(pageStore.isPaging());
    waitCleanup();
    assertFalse(lookupPageStore(ADDRESS).isPaging());
}

Example 4 with PagedReference

use of org.apache.activemq.artemis.core.paging.cursor.PagedReference in project activemq-artemis by apache.

the class QueueImpl method depage.

private void depage(final boolean scheduleExpiry) {
    depagePending = false;
    synchronized (this) {
        if (paused || pageIterator == null) {
            return;
        }
    }
    long maxSize = pageSubscription.getPagingStore().getPageSizeBytes();
    long timeout = System.currentTimeMillis() + DELIVERY_TIMEOUT;
    if (logger.isTraceEnabled()) {
        logger.trace("QueueMemorySize before depage on queue=" + this.getName() + " is " + queueMemorySize.get());
    }
    this.directDeliver = false;
    int depaged = 0;
    while (timeout > System.currentTimeMillis() && needsDepage() && pageIterator.hasNext()) {
        depaged++;
        PagedReference reference = pageIterator.next();
        if (logger.isTraceEnabled()) {
            logger.trace("Depaging reference " + reference + " on queue " + this.getName());
        }
        addTail(reference, false);
        pageIterator.remove();
        // We have to increment this here instead of in the iterator so we have access to the reference from next()
        pageSubscription.incrementDeliveredSize(getPersistentSize(reference));
    }
    if (logger.isDebugEnabled()) {
        if (depaged == 0 && queueMemorySize.get() >= maxSize) {
            logger.debug("Couldn't depage any message as the maxSize on the queue was achieved. " + "There are too many pending messages to be acked in reference to the page configuration");
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Queue Memory Size after depage on queue=" + this.getName() + " is " + queueMemorySize.get() + " with maxSize = " + maxSize + ". Depaged " + depaged + " messages, pendingDelivery=" + messageReferences.size() + ", intermediateMessageReferences= " + intermediateMessageReferences.size() + ", queueDelivering=" + deliveringMetrics.getMessageCount());
        }
    }
    deliverAsync();
    if (depaged > 0 && scheduleExpiry) {
        // This will just call an executor
        expireReferences();
    }
}

Example 5 with PagedReference

use of org.apache.activemq.artemis.core.paging.cursor.PagedReference in project activemq-artemis by apache.

the class PageCursorStressTest method testRestartWithHoleOnAck.

@Test
public void testRestartWithHoleOnAck() throws Exception {
    final int NUM_MESSAGES = 1000;
    int numberOfPages = addMessages(NUM_MESSAGES, 10 * 1024);
    System.out.println("Number of pages = " + numberOfPages);
    PageCursorProvider cursorProvider = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider();
    System.out.println("cursorProvider = " + cursorProvider);
    PageSubscription cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    System.out.println("Cursor: " + cursor);
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    for (int i = 0; i < 100; i++) {
        PagedReference msg = iterator.next();
        assertEquals(i, msg.getMessage().getIntProperty("key").intValue());
        if (i < 10 || i > 20) {
            cursor.ack(msg);
        }
    }
    server.getStorageManager().waitOnOperations();
    server.stop();
    OperationContextImpl.clearContext();
    server.start();
    cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    iterator = cursor.iterator();
    for (int i = 10; i <= 20; i++) {
        PagedReference msg = iterator.next();
        assertEquals(i, msg.getMessage().getIntProperty("key").intValue());
        cursor.ack(msg);
    }
    for (int i = 100; i < NUM_MESSAGES; i++) {
        PagedReference msg = iterator.next();
        assertEquals(i, msg.getMessage().getIntProperty("key").intValue());
        cursor.ack(msg);
    }
    server.stop();
    createServer();
    waitCleanup();
    assertEquals(1, lookupPageStore(ADDRESS).getNumberOfPages());
}