Examples with Page org.apache.activemq.artemis.core.paging.impl.Page used on opensource projects

Example 6 with Page

use of org.apache.activemq.artemis.core.paging.impl.Page in project activemq-artemis by apache.

the class ReplicationEndpoint method stop.

@Override
public synchronized void stop() throws Exception {
    if (!started) {
        return;
    }
    logger.trace("Stopping endpoint");
    started = false;
    OrderedExecutorFactory.flushExecutor(executor);
    // Channel may be null if there isn't a connection to a live server
    if (channel != null) {
        channel.close();
    }
    for (ReplicatedLargeMessage largeMessage : largeMessages.values()) {
        largeMessage.releaseResources();
    }
    largeMessages.clear();
    for (Entry<JournalContent, Map<Long, JournalSyncFile>> entry : filesReservedForSync.entrySet()) {
        for (JournalSyncFile filesReserved : entry.getValue().values()) {
            filesReserved.close();
        }
    }
    filesReservedForSync.clear();
    if (journals != null) {
        for (Journal j : journals) {
            if (j instanceof FileWrapperJournal)
                j.stop();
        }
    }
    for (ConcurrentMap<Integer, Page> map : pageIndex.values()) {
        for (Page page : map.values()) {
            try {
                page.sync();
                page.close(false);
            } catch (Exception e) {
                ActiveMQServerLogger.LOGGER.errorClosingPageOnReplication(e);
            }
        }
    }
    pageManager.stop();
    pageIndex.clear();
    // Storage needs to be the last to stop
    storageManager.stop();
    started = false;
}

Example 7 with Page

use of org.apache.activemq.artemis.core.paging.impl.Page in project activemq-artemis by apache.

the class XmlDataExporter method printPagedMessagesAsXML.

/**
 * Reads from the page files and prints messages as it finds them (making sure to check acks and transactions
 * from the journal).
 */
private void printPagedMessagesAsXML() {
    try {
        pagingmanager.start();
        SimpleString[] stores = pagingmanager.getStoreNames();
        for (SimpleString store : stores) {
            PagingStore pageStore = pagingmanager.getPageStore(store);
            if (pageStore != null) {
                File folder = pageStore.getFolder();
                ActiveMQServerLogger.LOGGER.debug("Reading page store " + store + " folder = " + folder);
                int pageId = (int) pageStore.getFirstPage();
                for (int i = 0; i < pageStore.getNumberOfPages(); i++) {
                    ActiveMQServerLogger.LOGGER.debug("Reading page " + pageId);
                    Page page = pageStore.createPage(pageId);
                    page.open();
                    List<PagedMessage> messages = page.read(storageManager);
                    page.close();
                    int messageId = 0;
                    for (PagedMessage message : messages) {
                        message.initMessage(storageManager);
                        long[] queueIDs = message.getQueueIDs();
                        List<String> queueNames = new ArrayList<>();
                        for (long queueID : queueIDs) {
                            PagePosition posCheck = new PagePositionImpl(pageId, messageId);
                            boolean acked = false;
                            Set<PagePosition> positions = cursorRecords.get(queueID);
                            if (positions != null) {
                                acked = positions.contains(posCheck);
                            }
                            if (!acked) {
                                PersistentQueueBindingEncoding queueBinding = queueBindings.get(queueID);
                                if (queueBinding != null) {
                                    SimpleString queueName = queueBinding.getQueueName();
                                    queueNames.add(queueName.toString());
                                }
                            }
                        }
                        if (queueNames.size() > 0 && (message.getTransactionID() == -1 || pgTXs.contains(message.getTransactionID()))) {
                            printSingleMessageAsXML(message.getMessage().toCore(), queueNames);
                        }
                        messageId++;
                    }
                    pageId++;
                }
            } else {
                ActiveMQServerLogger.LOGGER.debug("Page store was null");
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 8 with Page

use of org.apache.activemq.artemis.core.paging.impl.Page in project activemq-artemis by apache.

the class PagingStoreImplTest method testConcurrentPaging.

protected void testConcurrentPaging(final SequentialFileFactory factory, final int numberOfThreads) throws Exception {
    PagingStoreFactory storeFactory = new FakeStoreFactory(factory);
    final int MAX_SIZE = 1024 * 10;
    final AtomicLong messageIdGenerator = new AtomicLong(0);
    final AtomicInteger aliveProducers = new AtomicInteger(numberOfThreads);
    final CountDownLatch latchStart = new CountDownLatch(numberOfThreads);
    final ConcurrentHashMap<Long, Message> buffers = new ConcurrentHashMap<>();
    final ArrayList<Page> readPages = new ArrayList<>();
    AddressSettings settings = new AddressSettings().setPageSizeBytes(MAX_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
    final PagingStore storeImpl = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, new SimpleString("test"), settings, getExecutorFactory().getExecutor(), true);
    storeImpl.start();
    Assert.assertEquals(0, storeImpl.getNumberOfPages());
    // Marked the store to be paged
    storeImpl.startPaging();
    Assert.assertEquals(1, storeImpl.getNumberOfPages());
    final SimpleString destination = new SimpleString("test");
    class WriterThread extends Thread {

        Exception e;

        @Override
        public void run() {
            try {
                boolean firstTime = true;
                while (true) {
                    long id = messageIdGenerator.incrementAndGet();
                    // Each thread will Keep paging until all the messages are depaged.
                    // This is possible because the depage thread is not actually reading the pages.
                    // Just using the internal API to remove it from the page file system
                    Message msg = createMessage(id, storeImpl, destination, createRandomBuffer(id, 5));
                    final RoutingContextImpl ctx2 = new RoutingContextImpl(null);
                    if (storeImpl.page(msg, ctx2.getTransaction(), ctx2.getContextListing(storeImpl.getStoreName()), lock)) {
                        buffers.put(id, msg);
                    } else {
                        break;
                    }
                    if (firstTime) {
                        // We have at least one data paged. So, we can start depaging now
                        latchStart.countDown();
                        firstTime = false;
                    }
                }
            } catch (Exception e1) {
                e1.printStackTrace();
                this.e = e1;
            } finally {
                aliveProducers.decrementAndGet();
            }
        }
    }
    final class ReaderThread extends Thread {

        Exception e;

        @Override
        public void run() {
            try {
                // Wait every producer to produce at least one message
                ActiveMQTestBase.waitForLatch(latchStart);
                while (aliveProducers.get() > 0) {
                    Page page = storeImpl.depage();
                    if (page != null) {
                        readPages.add(page);
                    }
                }
            } catch (Exception e1) {
                e1.printStackTrace();
                this.e = e1;
            }
        }
    }
    WriterThread[] producerThread = new WriterThread[numberOfThreads];
    for (int i = 0; i < numberOfThreads; i++) {
        producerThread[i] = new WriterThread();
        producerThread[i].start();
    }
    ReaderThread consumer = new ReaderThread();
    consumer.start();
    for (int i = 0; i < numberOfThreads; i++) {
        producerThread[i].join();
        if (producerThread[i].e != null) {
            throw producerThread[i].e;
        }
    }
    consumer.join();
    if (consumer.e != null) {
        throw consumer.e;
    }
    final ConcurrentMap<Long, Message> buffers2 = new ConcurrentHashMap<>();
    for (Page page : readPages) {
        page.open();
        List<PagedMessage> msgs = page.read(new NullStorageManager());
        page.close();
        for (PagedMessage msg : msgs) {
            long id = msg.getMessage().toCore().getBodyBuffer().readLong();
            msg.getMessage().toCore().getBodyBuffer().resetReaderIndex();
            Message msgWritten = buffers.remove(id);
            buffers2.put(id, msg.getMessage());
            Assert.assertNotNull(msgWritten);
            Assert.assertEquals(msg.getMessage().getAddress(), msgWritten.getAddress());
            ActiveMQTestBase.assertEqualsBuffers(10, msgWritten.toCore().getBodyBuffer(), msg.getMessage().toCore().getBodyBuffer());
        }
    }
    Assert.assertEquals(0, buffers.size());
    List<String> files = factory.listFiles("page");
    Assert.assertTrue(files.size() != 0);
    for (String file : files) {
        SequentialFile fileTmp = factory.createSequentialFile(file);
        fileTmp.open();
        Assert.assertTrue("The page file size (" + fileTmp.size() + ") shouldn't be > " + MAX_SIZE, fileTmp.size() <= MAX_SIZE);
        fileTmp.close();
    }
    PagingStore storeImpl2 = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, new SimpleString("test"), settings, getExecutorFactory().getExecutor(), true);
    storeImpl2.start();
    int numberOfPages = storeImpl2.getNumberOfPages();
    Assert.assertTrue(numberOfPages != 0);
    storeImpl2.startPaging();
    storeImpl2.startPaging();
    Assert.assertEquals(numberOfPages, storeImpl2.getNumberOfPages());
    long lastMessageId = messageIdGenerator.incrementAndGet();
    Message lastMsg = createMessage(lastMessageId, storeImpl, destination, createRandomBuffer(lastMessageId, 5));
    storeImpl2.forceAnotherPage();
    final RoutingContextImpl ctx = new RoutingContextImpl(null);
    storeImpl2.page(lastMsg, ctx.getTransaction(), ctx.getContextListing(storeImpl2.getStoreName()), lock);
    buffers2.put(lastMessageId, lastMsg);
    Page lastPage = null;
    while (true) {
        Page page = storeImpl2.depage();
        if (page == null) {
            break;
        }
        lastPage = page;
        page.open();
        List<PagedMessage> msgs = page.read(new NullStorageManager());
        page.close();
        for (PagedMessage msg : msgs) {
            long id = msg.getMessage().toCore().getBodyBuffer().readLong();
            Message msgWritten = buffers2.remove(id);
            Assert.assertNotNull(msgWritten);
            Assert.assertEquals(msg.getMessage().getAddress(), msgWritten.getAddress());
            ActiveMQTestBase.assertEqualsByteArrays(msgWritten.toCore().getBodyBuffer().writerIndex(), msgWritten.toCore().getBodyBuffer().toByteBuffer().array(), msg.getMessage().toCore().getBodyBuffer().toByteBuffer().array());
        }
    }
    lastPage.open();
    List<PagedMessage> lastMessages = lastPage.read(new NullStorageManager());
    lastPage.close();
    Assert.assertEquals(1, lastMessages.size());
    lastMessages.get(0).getMessage().toCore().getBodyBuffer().resetReaderIndex();
    Assert.assertEquals(lastMessages.get(0).getMessage().toCore().getBodyBuffer().readLong(), lastMessageId);
    Assert.assertEquals(0, buffers2.size());
    Assert.assertEquals(0, storeImpl.getAddressSize());
}

Example 9 with Page

use of org.apache.activemq.artemis.core.paging.impl.Page in project activemq-artemis by apache.

the class PostOfficeJournalLoader method recoverPendingPageCounters.

/**
 * This method will recover the counters after failures making sure the page counter doesn't get out of sync
 *
 * @param pendingNonTXPageCounter
 * @throws Exception
 */
@Override
public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter) throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need
    Transaction txRecoverCounter = new TransactionImpl(storageManager);
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);
    for (Map.Entry<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> addressPageMapEntry : perAddressMap.entrySet()) {
        PagingStore store = pagingManager.getPageStore(addressPageMapEntry.getKey());
        Map<Long, Map<Long, List<PageCountPending>>> perPageMap = addressPageMapEntry.getValue();
        // We have already generated this before, so it can't be null
        assert (perPageMap != null);
        for (Long pageId : perPageMap.keySet()) {
            Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);
            // This can't be true!
            assert (perQueue != null);
            if (store.checkPageFileExists(pageId.intValue())) {
                // on this case we need to recalculate the records
                Page pg = store.createPage(pageId.intValue());
                pg.open();
                List<PagedMessage> pgMessages = pg.read(storageManager);
                Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();
                Map<Long, AtomicLong> sizePerQueueOnPage = new HashMap<>();
                for (PagedMessage pgd : pgMessages) {
                    if (pgd.getTransactionID() <= 0) {
                        for (long q : pgd.getQueueIDs()) {
                            AtomicInteger countQ = countsPerQueueOnPage.get(q);
                            AtomicLong sizeQ = sizePerQueueOnPage.get(q);
                            if (countQ == null) {
                                countQ = new AtomicInteger(0);
                                countsPerQueueOnPage.put(q, countQ);
                            }
                            if (sizeQ == null) {
                                sizeQ = new AtomicLong(0);
                                sizePerQueueOnPage.put(q, sizeQ);
                            }
                            countQ.incrementAndGet();
                            if (pgd.getPersistentSize() > 0) {
                                sizeQ.addAndGet(pgd.getPersistentSize());
                            }
                        }
                    }
                }
                for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
                    for (PageCountPending record : entry.getValue()) {
                        logger.debug("Deleting pg tempCount " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                    }
                    PageSubscriptionCounter counter = store.getCursorProvider().getSubscription(entry.getKey()).getCounter();
                    AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());
                    AtomicLong sizeValue = sizePerQueueOnPage.get(entry.getKey());
                    if (value == null) {
                        logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
                    } else {
                        logger.debug("Replacing counter " + value.get());
                        counter.increment(txRecoverCounter, value.get(), sizeValue.get());
                    }
                }
            } else {
                // on this case the page file didn't exist, we just remove all the records since the page is already gone
                logger.debug("Page " + pageId + " didn't exist on address " + addressPageMapEntry.getKey() + ", so we are just removing records");
                for (List<PageCountPending> records : perQueue.values()) {
                    for (PageCountPending record : records) {
                        logger.debug("Removing pending page counter " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                        txRecoverCounter.setContainsPersistent();
                    }
                }
            }
        }
    }
    txRecoverCounter.commit();
}

Example 10 with Page

use of org.apache.activemq.artemis.core.paging.impl.Page in project activemq-artemis by apache.

the class PrintData method printPages.

private static void printPages(DescribeJournal describeJournal, StorageManager sm, PagingManager manager, PrintStream out, boolean safe) throws Exception {
    PageCursorsInfo cursorACKs = calculateCursorsInfo(describeJournal.getRecords());
    Set<Long> pgTXs = cursorACKs.getPgTXs();
    manager.start();
    SimpleString[] stores = manager.getStoreNames();
    for (SimpleString store : stores) {
        PagingStore pgStore = manager.getPageStore(store);
        File folder = null;
        if (pgStore != null) {
            folder = pgStore.getFolder();
        }
        out.println("####################################################################################################");
        out.println("Exploring store " + store + " folder = " + folder);
        int pgid = (int) pgStore.getFirstPage();
        for (int pg = 0; pg < pgStore.getNumberOfPages(); pg++) {
            out.println("*******   Page " + pgid);
            Page page = pgStore.createPage(pgid);
            page.open();
            List<PagedMessage> msgs = page.read(sm);
            page.close();
            int msgID = 0;
            for (PagedMessage msg : msgs) {
                msg.initMessage(sm);
                if (safe) {
                    try {
                        out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ", msg=" + msg.getMessage().getClass().getSimpleName() + "(safe data, size=" + msg.getMessage().getPersistentSize() + ")");
                    } catch (Exception e) {
                        out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ", msg=" + msg.getMessage().getClass().getSimpleName() + "(safe data)");
                    }
                } else {
                    out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ",userMessageID=" + (msg.getMessage().getUserID() != null ? msg.getMessage().getUserID() : "") + ", msg=" + msg.getMessage());
                }
                out.print(",Queues = ");
                long[] q = msg.getQueueIDs();
                for (int i = 0; i < q.length; i++) {
                    out.print(q[i]);
                    PagePosition posCheck = new PagePositionImpl(pgid, msgID);
                    boolean acked = false;
                    Set<PagePosition> positions = cursorACKs.getCursorRecords().get(q[i]);
                    if (positions != null) {
                        acked = positions.contains(posCheck);
                    }
                    if (acked) {
                        out.print(" (ACK)");
                    }
                    if (cursorACKs.getCompletePages(q[i]).contains(Long.valueOf(pgid))) {
                        out.println(" (PG-COMPLETE)");
                    }
                    if (i + 1 < q.length) {
                        out.print(",");
                    }
                }
                if (msg.getTransactionID() >= 0 && !pgTXs.contains(msg.getTransactionID())) {
                    out.print(", **PG_TX_NOT_FOUND**");
                }
                out.println();
                msgID++;
            }
            pgid++;
        }
    }
}