Examples with PagingStoreImpl org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl used on opensource projects

Example 1 with PagingStoreImpl

use of org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl in project activemq-artemis by apache.

the class PagingStoreImplTest method testRestartPage.

@Test
public void testRestartPage() throws Throwable {
    clearDataRecreateServerDirs();
    SequentialFileFactory factory = new NIOSequentialFileFactory(new File(getPageDir()), 1);
    PagingStoreFactory storeFactory = new FakeStoreFactory(factory);
    final int MAX_SIZE = 1024 * 10;
    AddressSettings settings = new AddressSettings().setPageSizeBytes(MAX_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
    final PagingStore storeImpl = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, new SimpleString("test"), settings, getExecutorFactory().getExecutor(), true);
    storeImpl.start();
    Assert.assertEquals(0, storeImpl.getNumberOfPages());
    // Marked the store to be paged
    storeImpl.startPaging();
    storeImpl.depage();
    Assert.assertNull(storeImpl.getCurrentPage());
    storeImpl.startPaging();
    Assert.assertNotNull(storeImpl.getCurrentPage());
    storeImpl.stop();
}

Example 2 with PagingStoreImpl

use of org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl in project activemq-artemis by apache.

the class PagingStoreImplTest method testConcurrentPaging.

protected void testConcurrentPaging(final SequentialFileFactory factory, final int numberOfThreads) throws Exception {
    PagingStoreFactory storeFactory = new FakeStoreFactory(factory);
    final int MAX_SIZE = 1024 * 10;
    final AtomicLong messageIdGenerator = new AtomicLong(0);
    final AtomicInteger aliveProducers = new AtomicInteger(numberOfThreads);
    final CountDownLatch latchStart = new CountDownLatch(numberOfThreads);
    final ConcurrentHashMap<Long, Message> buffers = new ConcurrentHashMap<>();
    final ArrayList<Page> readPages = new ArrayList<>();
    AddressSettings settings = new AddressSettings().setPageSizeBytes(MAX_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
    final PagingStore storeImpl = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, new SimpleString("test"), settings, getExecutorFactory().getExecutor(), true);
    storeImpl.start();
    Assert.assertEquals(0, storeImpl.getNumberOfPages());
    // Marked the store to be paged
    storeImpl.startPaging();
    Assert.assertEquals(1, storeImpl.getNumberOfPages());
    final SimpleString destination = new SimpleString("test");
    class WriterThread extends Thread {

        Exception e;

        @Override
        public void run() {
            try {
                boolean firstTime = true;
                while (true) {
                    long id = messageIdGenerator.incrementAndGet();
                    // Each thread will Keep paging until all the messages are depaged.
                    // This is possible because the depage thread is not actually reading the pages.
                    // Just using the internal API to remove it from the page file system
                    Message msg = createMessage(id, storeImpl, destination, createRandomBuffer(id, 5));
                    final RoutingContextImpl ctx2 = new RoutingContextImpl(null);
                    if (storeImpl.page(msg, ctx2.getTransaction(), ctx2.getContextListing(storeImpl.getStoreName()), lock)) {
                        buffers.put(id, msg);
                    } else {
                        break;
                    }
                    if (firstTime) {
                        // We have at least one data paged. So, we can start depaging now
                        latchStart.countDown();
                        firstTime = false;
                    }
                }
            } catch (Exception e1) {
                e1.printStackTrace();
                this.e = e1;
            } finally {
                aliveProducers.decrementAndGet();
            }
        }
    }
    final class ReaderThread extends Thread {

        Exception e;

        @Override
        public void run() {
            try {
                // Wait every producer to produce at least one message
                ActiveMQTestBase.waitForLatch(latchStart);
                while (aliveProducers.get() > 0) {
                    Page page = storeImpl.depage();
                    if (page != null) {
                        readPages.add(page);
                    }
                }
            } catch (Exception e1) {
                e1.printStackTrace();
                this.e = e1;
            }
        }
    }
    WriterThread[] producerThread = new WriterThread[numberOfThreads];
    for (int i = 0; i < numberOfThreads; i++) {
        producerThread[i] = new WriterThread();
        producerThread[i].start();
    }
    ReaderThread consumer = new ReaderThread();
    consumer.start();
    for (int i = 0; i < numberOfThreads; i++) {
        producerThread[i].join();
        if (producerThread[i].e != null) {
            throw producerThread[i].e;
        }
    }
    consumer.join();
    if (consumer.e != null) {
        throw consumer.e;
    }
    final ConcurrentMap<Long, Message> buffers2 = new ConcurrentHashMap<>();
    for (Page page : readPages) {
        page.open();
        List<PagedMessage> msgs = page.read(new NullStorageManager());
        page.close();
        for (PagedMessage msg : msgs) {
            long id = msg.getMessage().toCore().getBodyBuffer().readLong();
            msg.getMessage().toCore().getBodyBuffer().resetReaderIndex();
            Message msgWritten = buffers.remove(id);
            buffers2.put(id, msg.getMessage());
            Assert.assertNotNull(msgWritten);
            Assert.assertEquals(msg.getMessage().getAddress(), msgWritten.getAddress());
            ActiveMQTestBase.assertEqualsBuffers(10, msgWritten.toCore().getBodyBuffer(), msg.getMessage().toCore().getBodyBuffer());
        }
    }
    Assert.assertEquals(0, buffers.size());
    List<String> files = factory.listFiles("page");
    Assert.assertTrue(files.size() != 0);
    for (String file : files) {
        SequentialFile fileTmp = factory.createSequentialFile(file);
        fileTmp.open();
        Assert.assertTrue("The page file size (" + fileTmp.size() + ") shouldn't be > " + MAX_SIZE, fileTmp.size() <= MAX_SIZE);
        fileTmp.close();
    }
    PagingStore storeImpl2 = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, new SimpleString("test"), settings, getExecutorFactory().getExecutor(), true);
    storeImpl2.start();
    int numberOfPages = storeImpl2.getNumberOfPages();
    Assert.assertTrue(numberOfPages != 0);
    storeImpl2.startPaging();
    storeImpl2.startPaging();
    Assert.assertEquals(numberOfPages, storeImpl2.getNumberOfPages());
    long lastMessageId = messageIdGenerator.incrementAndGet();
    Message lastMsg = createMessage(lastMessageId, storeImpl, destination, createRandomBuffer(lastMessageId, 5));
    storeImpl2.forceAnotherPage();
    final RoutingContextImpl ctx = new RoutingContextImpl(null);
    storeImpl2.page(lastMsg, ctx.getTransaction(), ctx.getContextListing(storeImpl2.getStoreName()), lock);
    buffers2.put(lastMessageId, lastMsg);
    Page lastPage = null;
    while (true) {
        Page page = storeImpl2.depage();
        if (page == null) {
            break;
        }
        lastPage = page;
        page.open();
        List<PagedMessage> msgs = page.read(new NullStorageManager());
        page.close();
        for (PagedMessage msg : msgs) {
            long id = msg.getMessage().toCore().getBodyBuffer().readLong();
            Message msgWritten = buffers2.remove(id);
            Assert.assertNotNull(msgWritten);
            Assert.assertEquals(msg.getMessage().getAddress(), msgWritten.getAddress());
            ActiveMQTestBase.assertEqualsByteArrays(msgWritten.toCore().getBodyBuffer().writerIndex(), msgWritten.toCore().getBodyBuffer().toByteBuffer().array(), msg.getMessage().toCore().getBodyBuffer().toByteBuffer().array());
        }
    }
    lastPage.open();
    List<PagedMessage> lastMessages = lastPage.read(new NullStorageManager());
    lastPage.close();
    Assert.assertEquals(1, lastMessages.size());
    lastMessages.get(0).getMessage().toCore().getBodyBuffer().resetReaderIndex();
    Assert.assertEquals(lastMessages.get(0).getMessage().toCore().getBodyBuffer().readLong(), lastMessageId);
    Assert.assertEquals(0, buffers2.size());
    Assert.assertEquals(0, storeImpl.getAddressSize());
}

Example 3 with PagingStoreImpl

use of org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl in project activemq-artemis by apache.

the class PageCursorStressTest method testLazyCommit.

@Test
public void testLazyCommit() throws Exception {
    PagingStoreImpl pageStore = lookupPageStore(ADDRESS);
    pageStore.startPaging();
    final int NUM_MESSAGES = 100;
    final int messageSize = 100 * 1024;
    PageCursorProvider cursorProvider = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider();
    System.out.println("cursorProvider = " + cursorProvider);
    PageSubscription cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    System.out.println("Cursor: " + cursor);
    StorageManager storage = this.server.getStorageManager();
    long pgtxLazy = storage.generateID();
    Transaction txLazy = pgMessages(storage, pageStore, pgtxLazy, 0, NUM_MESSAGES, messageSize);
    addMessages(100, NUM_MESSAGES, messageSize);
    System.out.println("Number of pages - " + pageStore.getNumberOfPages());
    // First consume what's already there without any tx as nothing was committed
    for (int i = 100; i < 200; i++) {
        PagedReference pos = iterator.next();
        assertNotNull("Null at position " + i, pos);
        assertEquals(i, pos.getMessage().getIntProperty("key").intValue());
        cursor.ack(pos);
    }
    assertNull(iterator.next());
    txLazy.commit();
    storage.waitOnOperations();
    for (int i = 0; i < 100; i++) {
        PagedReference pos = iterator.next();
        assertNotNull("Null at position " + i, pos);
        assertEquals(i, pos.getMessage().getIntProperty("key").intValue());
        cursor.ack(pos);
    }
    assertNull(iterator.next());
    waitCleanup();
    server.stop();
    createServer();
    waitCleanup();
    assertEquals(1, lookupPageStore(ADDRESS).getNumberOfPages());
}

Example 4 with PagingStoreImpl

use of org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl in project activemq-artemis by apache.

the class PageCursorStressTest method testConsumeLivePage.

@Test
public void testConsumeLivePage() throws Exception {
    PagingStoreImpl pageStore = lookupPageStore(ADDRESS);
    pageStore.startPaging();
    final int NUM_MESSAGES = 100;
    final int messageSize = 1024 * 1024;
    PageCursorProvider cursorProvider = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider();
    System.out.println("cursorProvider = " + cursorProvider);
    PageSubscription cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    System.out.println("Cursor: " + cursor);
    RoutingContextImpl ctx = generateCTX();
    LinkedListIterator<PagedReference> iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES; i++) {
        // if (i % 100 == 0)
        System.out.println("read/written " + i);
        ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
        Message msg = new CoreMessage(i, buffer.writerIndex());
        msg.putIntProperty("key", i);
        msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
        Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
        assertNull(iterator.next());
    }
    OperationContextImpl.clearContext();
    ctx = generateCTX();
    pageStore = lookupPageStore(ADDRESS);
    cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES * 2; i++) {
        if (i % 100 == 0)
            System.out.println("Paged " + i);
        if (i >= NUM_MESSAGES) {
            ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
            Message msg = new CoreMessage(i, buffer.writerIndex());
            msg.putIntProperty("key", i);
            msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
            Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        }
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
    }
    OperationContextImpl.clearContext();
    pageStore = lookupPageStore(ADDRESS);
    cursor = this.server.getPagingManager().getPageStore(ADDRESS).getCursorProvider().getSubscription(queue.getID());
    iterator = cursor.iterator();
    for (int i = 0; i < NUM_MESSAGES * 3; i++) {
        if (i % 100 == 0)
            System.out.println("Paged " + i);
        if (i >= NUM_MESSAGES * 2 - 1) {
            ActiveMQBuffer buffer = RandomUtil.randomBuffer(messageSize, i + 1L);
            Message msg = new CoreMessage(i, buffer.writerIndex());
            msg.putIntProperty("key", i + 1);
            msg.getBodyBuffer().writeBytes(buffer, 0, buffer.writerIndex());
            Assert.assertTrue(pageStore.page(msg, ctx.getTransaction(), ctx.getContextListing(ADDRESS), lock));
        }
        PagedReference readMessage = iterator.next();
        assertNotNull(readMessage);
        cursor.ack(readMessage);
        assertEquals(i, readMessage.getMessage().getIntProperty("key").intValue());
    }
    PagedReference readMessage = iterator.next();
    assertEquals(NUM_MESSAGES * 3, readMessage.getMessage().getIntProperty("key").intValue());
    cursor.ack(readMessage);
    server.getStorageManager().waitOnOperations();
    pageStore.flushExecutors();
    assertFalse(pageStore.isPaging());
    server.stop();
    createServer();
    assertFalse(pageStore.isPaging());
    waitCleanup();
    assertFalse(lookupPageStore(ADDRESS).isPaging());
}

Example 5 with PagingStoreImpl

use of org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl in project activemq-artemis by apache.

the class PagingStoreImplTest method testDepageOnCurrentPage.

@Test
public void testDepageOnCurrentPage() throws Exception {
    SequentialFileFactory factory = new FakeSequentialFileFactory();
    SimpleString destination = new SimpleString("test");
    PagingStoreFactory storeFactory = new FakeStoreFactory(factory);
    PagingStoreImpl storeImpl = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, PagingStoreImplTest.destinationTestName, new AddressSettings().setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE), getExecutorFactory().getExecutor(), true);
    storeImpl.start();
    Assert.assertEquals(0, storeImpl.getNumberOfPages());
    storeImpl.startPaging();
    List<ActiveMQBuffer> buffers = new ArrayList<>();
    int numMessages = 10;
    for (int i = 0; i < numMessages; i++) {
        ActiveMQBuffer buffer = createRandomBuffer(i + 1L, 10);
        buffers.add(buffer);
        Message msg = createMessage(i, storeImpl, destination, buffer);
        final RoutingContextImpl ctx = new RoutingContextImpl(null);
        Assert.assertTrue(storeImpl.page(msg, ctx.getTransaction(), ctx.getContextListing(storeImpl.getStoreName()), lock));
    }
    Assert.assertEquals(1, storeImpl.getNumberOfPages());
    storeImpl.sync();
    Page page = storeImpl.depage();
    page.open();
    List<PagedMessage> msg = page.read(new NullStorageManager());
    Assert.assertEquals(numMessages, msg.size());
    Assert.assertEquals(1, storeImpl.getNumberOfPages());
    page = storeImpl.depage();
    Assert.assertNull(page);
    Assert.assertEquals(0, storeImpl.getNumberOfPages());
    for (int i = 0; i < numMessages; i++) {
        ActiveMQBuffer horn1 = buffers.get(i);
        ActiveMQBuffer horn2 = msg.get(i).getMessage().toCore().getBodyBuffer();
        horn1.resetReaderIndex();
        horn2.resetReaderIndex();
        for (int j = 0; j < horn1.writerIndex(); j++) {
            Assert.assertEquals(horn1.readByte(), horn2.readByte());
        }
    }
}