Examples with PagingStore org.apache.activemq.artemis.core.paging.PagingStore used on opensource projects

Example 11 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PostOfficeJournalLoader method recoverPendingPageCounters.

/**
 * This method will recover the counters after failures making sure the page counter doesn't get out of sync
 *
 * @param pendingNonTXPageCounter
 * @throws Exception
 */
@Override
public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter) throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need
    Transaction txRecoverCounter = new TransactionImpl(storageManager);
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);
    for (Map.Entry<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> addressPageMapEntry : perAddressMap.entrySet()) {
        PagingStore store = pagingManager.getPageStore(addressPageMapEntry.getKey());
        Map<Long, Map<Long, List<PageCountPending>>> perPageMap = addressPageMapEntry.getValue();
        // We have already generated this before, so it can't be null
        assert (perPageMap != null);
        for (Long pageId : perPageMap.keySet()) {
            Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);
            // This can't be true!
            assert (perQueue != null);
            if (store.checkPageFileExists(pageId.intValue())) {
                // on this case we need to recalculate the records
                Page pg = store.createPage(pageId.intValue());
                pg.open();
                List<PagedMessage> pgMessages = pg.read(storageManager);
                Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();
                Map<Long, AtomicLong> sizePerQueueOnPage = new HashMap<>();
                for (PagedMessage pgd : pgMessages) {
                    if (pgd.getTransactionID() <= 0) {
                        for (long q : pgd.getQueueIDs()) {
                            AtomicInteger countQ = countsPerQueueOnPage.get(q);
                            AtomicLong sizeQ = sizePerQueueOnPage.get(q);
                            if (countQ == null) {
                                countQ = new AtomicInteger(0);
                                countsPerQueueOnPage.put(q, countQ);
                            }
                            if (sizeQ == null) {
                                sizeQ = new AtomicLong(0);
                                sizePerQueueOnPage.put(q, sizeQ);
                            }
                            countQ.incrementAndGet();
                            if (pgd.getPersistentSize() > 0) {
                                sizeQ.addAndGet(pgd.getPersistentSize());
                            }
                        }
                    }
                }
                for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
                    for (PageCountPending record : entry.getValue()) {
                        logger.debug("Deleting pg tempCount " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                    }
                    PageSubscriptionCounter counter = store.getCursorProvider().getSubscription(entry.getKey()).getCounter();
                    AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());
                    AtomicLong sizeValue = sizePerQueueOnPage.get(entry.getKey());
                    if (value == null) {
                        logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
                    } else {
                        logger.debug("Replacing counter " + value.get());
                        counter.increment(txRecoverCounter, value.get(), sizeValue.get());
                    }
                }
            } else {
                // on this case the page file didn't exist, we just remove all the records since the page is already gone
                logger.debug("Page " + pageId + " didn't exist on address " + addressPageMapEntry.getKey() + ", so we are just removing records");
                for (List<PageCountPending> records : perQueue.values()) {
                    for (PageCountPending record : records) {
                        logger.debug("Removing pending page counter " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                        txRecoverCounter.setContainsPersistent();
                    }
                }
            }
        }
    }
    txRecoverCounter.commit();
}

Example 12 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class FileMoveManagerTest method testMoveOverPaging.

@Test
public void testMoveOverPaging() throws Exception {
    AssertionLoggerHandler.startCapture();
    ExecutorService threadPool = Executors.newCachedThreadPool();
    try {
        manager.setMaxFolders(3);
        for (int i = 1; i <= 10; i++) {
            HierarchicalRepository<AddressSettings> addressSettings = new HierarchicalObjectRepository<>();
            AddressSettings settings = new AddressSettings();
            settings.setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
            addressSettings.setDefault(settings);
            final StorageManager storageManager = new NullStorageManager();
            PagingStoreFactoryNIO storeFactory = new PagingStoreFactoryNIO(storageManager, dataLocation, 100, null, new OrderedExecutorFactory(threadPool), true, null);
            PagingManagerImpl managerImpl = new PagingManagerImpl(storeFactory, addressSettings, -1);
            managerImpl.start();
            PagingStore store = managerImpl.getPageStore(new SimpleString("simple-test"));
            store.startPaging();
            store.stop();
            managerImpl.stop();
            manager.doMove();
            Assert.assertEquals(Math.min(i, manager.getMaxFolders()), manager.getNumberOfFolders());
        }
        Assert.assertFalse("The loggers are complaining about address.txt", AssertionLoggerHandler.findText("address.txt"));
    } finally {
        AssertionLoggerHandler.stopCapture();
        threadPool.shutdown();
    }
}

Example 13 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class ServerInfo method appendPagingInfos.

private String appendPagingInfos() {
    StringBuilder info = new StringBuilder();
    for (SimpleString storeName : pagingManager.getStoreNames()) {
        PagingStore pageStore;
        try {
            pageStore = pagingManager.getPageStore(storeName);
            info.append(String.format("\t%s: %s%n", storeName, SizeFormatterUtil.sizeof(pageStore.getPageSizeBytes() * pageStore.getNumberOfPages())));
        } catch (Exception e) {
            info.append(String.format("\t%s: %s%n", storeName, e.getMessage()));
        }
    }
    return info.toString();
}

Example 14 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method testTwoQueues.

@Test
public void testTwoQueues() throws Exception {
    boolean persistentMessages = true;
    clearDataRecreateServerDirs();
    Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
    server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
    server.start();
    final int messageSize = 1024;
    final int numberOfMessages = 1000;
    try {
        ServerLocator locator = createInVMNonHALocator().setClientFailureCheckPeriod(120000).setConnectionTTL(5000000).setCallTimeout(120000).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
        ClientSessionFactory sf = locator.createSessionFactory();
        ClientSession session = sf.createSession(false, false, false);
        session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS.concat("=1"), null, true);
        session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS.concat("=2"), null, true);
        ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
        ClientMessage message = null;
        byte[] body = new byte[messageSize];
        for (int i = 0; i < numberOfMessages; i++) {
            message = session.createMessage(persistentMessages);
            ActiveMQBuffer bodyLocal = message.getBodyBuffer();
            bodyLocal.writeBytes(body);
            message.putIntProperty("propTest", i % 2 == 0 ? 1 : 2);
            producer.send(message);
            if (i % 1000 == 0) {
                session.commit();
            }
        }
        session.commit();
        session.start();
        for (int msg = 1; msg <= 2; msg++) {
            ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS.concat("=" + msg));
            for (int i = 0; i < numberOfMessages; i++) {
                message = consumer.receive(5000);
                assertNotNull(message);
                message.acknowledge();
                // assertEquals(msg, message.getIntProperty("propTest").intValue());
                System.out.println("i = " + i + " msg = " + message.getIntProperty("propTest"));
            }
            session.commit();
            assertNull(consumer.receiveImmediate());
            consumer.close();
        }
        PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
        store.getCursorProvider().cleanup();
        long timeout = System.currentTimeMillis() + 5000;
        while (store.isPaging() && timeout > System.currentTimeMillis()) {
            Thread.sleep(100);
        }
        store.getCursorProvider().cleanup();
        waitForNotPaging(server.locateQueue(PagingTest.ADDRESS.concat("=1")));
        sf.close();
        locator.close();
    } finally {
        try {
            server.stop();
        } catch (Throwable ignored) {
        }
    }
}

Example 15 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method testRestartWithComplete.

// The pages are complete, and this is simulating a scenario where the server crashed before deleting the pages.
@Test
public void testRestartWithComplete() throws Exception {
    clearDataRecreateServerDirs();
    Configuration config = createDefaultInVMConfig();
    final AtomicBoolean mainCleanup = new AtomicBoolean(true);
    class InterruptedCursorProvider extends PageCursorProviderImpl {

        InterruptedCursorProvider(PagingStore pagingStore, StorageManager storageManager, ArtemisExecutor executor, int maxCacheSize) {
            super(pagingStore, storageManager, executor, maxCacheSize);
        }

        @Override
        public void cleanup() {
            if (mainCleanup.get()) {
                super.cleanup();
            } else {
                try {
                    pagingStore.unlock();
                } catch (Throwable ignored) {
                }
            }
        }
    }
    server = new ActiveMQServerImpl(config, ManagementFactory.getPlatformMBeanServer(), new ActiveMQSecurityManagerImpl()) {

        @Override
        protected PagingStoreFactoryNIO getPagingStoreFactory() {
            return new PagingStoreFactoryNIO(this.getStorageManager(), this.getConfiguration().getPagingLocation(), this.getConfiguration().getJournalBufferTimeout_NIO(), this.getScheduledPool(), this.getExecutorFactory(), this.getConfiguration().isJournalSyncNonTransactional(), null) {

                @Override
                public PageCursorProvider newCursorProvider(PagingStore store, StorageManager storageManager, AddressSettings addressSettings, ArtemisExecutor executor) {
                    return new InterruptedCursorProvider(store, storageManager, executor, addressSettings.getPageCacheMaxSize());
                }
            };
        }
    };
    addServer(server);
    AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PagingTest.PAGE_SIZE).setMaxSizeBytes(PagingTest.PAGE_MAX).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
    server.getAddressSettingsRepository().addMatch("#", defaultSetting);
    server.start();
    locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
    sf = createSessionFactory(locator);
    ClientSession session = sf.createSession(true, true, 0);
    session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
    Queue queue = server.locateQueue(ADDRESS);
    queue.getPageSubscription().getPagingStore().startPaging();
    ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
    ClientMessage message;
    for (int i = 0; i < 20; i++) {
        message = session.createMessage(true);
        ActiveMQBuffer bodyLocal = message.getBodyBuffer();
        bodyLocal.writeBytes(new byte[100 * 4]);
        message.putIntProperty(new SimpleString("idi"), i);
        producer.send(message);
        session.commit();
        if (i < 19) {
            queue.getPageSubscription().getPagingStore().forceAnotherPage();
        }
    }
    Assert.assertEquals(20, queue.getPageSubscription().getPagingStore().getCurrentWritingPage());
    // This will force a scenario where the pages are cleaned up. When restarting we need to check if the current page is complete
    // if it is complete we must move to another page avoiding races on cleanup
    // which could happen during a crash / restart
    long tx = server.getStorageManager().generateID();
    for (int i = 1; i <= 20; i++) {
        server.getStorageManager().storePageCompleteTransactional(tx, queue.getID(), new PagePositionImpl(i, 1));
    }
    server.getStorageManager().commit(tx);
    session.close();
    sf.close();
    server.stop();
    mainCleanup.set(false);
    logger.trace("Server restart");
    server.start();
    queue = server.locateQueue(ADDRESS);
    locator = createInVMNonHALocator();
    sf = createSessionFactory(locator);
    session = sf.createSession(null, null, false, false, true, false, 0);
    producer = session.createProducer(PagingTest.ADDRESS);
    for (int i = 0; i < 10; i++) {
        message = session.createMessage(true);
        ActiveMQBuffer bodyLocal = message.getBodyBuffer();
        bodyLocal.writeBytes(new byte[100 * 4]);
        message.putIntProperty(new SimpleString("newid"), i);
        producer.send(message);
        session.commit();
        if (i == 5) {
            queue.getPageSubscription().getPagingStore().forceAnotherPage();
        }
    }
    mainCleanup.set(true);
    queue = server.locateQueue(ADDRESS);
    queue.getPageSubscription().cleanupEntries(false);
    queue.getPageSubscription().getPagingStore().getCursorProvider().cleanup();
    ClientConsumer consumer = session.createConsumer(ADDRESS);
    session.start();
    for (int i = 0; i < 10; i++) {
        message = consumer.receive(5000);
        Assert.assertNotNull(message);
        Assert.assertEquals(i, message.getIntProperty("newid").intValue());
        message.acknowledge();
    }
    server.stop();
// Thread.sleep(5000);
}