Examples with PagedMessage org.apache.activemq.artemis.core.paging.PagedMessage used on opensource projects

Example 6 with PagedMessage

use of org.apache.activemq.artemis.core.paging.PagedMessage in project activemq-artemis by apache.

the class PostOfficeJournalLoader method recoverPendingPageCounters.

/**
 * This method will recover the counters after failures making sure the page counter doesn't get out of sync
 *
 * @param pendingNonTXPageCounter
 * @throws Exception
 */
@Override
public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter) throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need
    Transaction txRecoverCounter = new TransactionImpl(storageManager);
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);
    for (Map.Entry<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> addressPageMapEntry : perAddressMap.entrySet()) {
        PagingStore store = pagingManager.getPageStore(addressPageMapEntry.getKey());
        Map<Long, Map<Long, List<PageCountPending>>> perPageMap = addressPageMapEntry.getValue();
        // We have already generated this before, so it can't be null
        assert (perPageMap != null);
        for (Long pageId : perPageMap.keySet()) {
            Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);
            // This can't be true!
            assert (perQueue != null);
            if (store.checkPageFileExists(pageId.intValue())) {
                // on this case we need to recalculate the records
                Page pg = store.createPage(pageId.intValue());
                pg.open();
                List<PagedMessage> pgMessages = pg.read(storageManager);
                Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();
                Map<Long, AtomicLong> sizePerQueueOnPage = new HashMap<>();
                for (PagedMessage pgd : pgMessages) {
                    if (pgd.getTransactionID() <= 0) {
                        for (long q : pgd.getQueueIDs()) {
                            AtomicInteger countQ = countsPerQueueOnPage.get(q);
                            AtomicLong sizeQ = sizePerQueueOnPage.get(q);
                            if (countQ == null) {
                                countQ = new AtomicInteger(0);
                                countsPerQueueOnPage.put(q, countQ);
                            }
                            if (sizeQ == null) {
                                sizeQ = new AtomicLong(0);
                                sizePerQueueOnPage.put(q, sizeQ);
                            }
                            countQ.incrementAndGet();
                            if (pgd.getPersistentSize() > 0) {
                                sizeQ.addAndGet(pgd.getPersistentSize());
                            }
                        }
                    }
                }
                for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
                    for (PageCountPending record : entry.getValue()) {
                        logger.debug("Deleting pg tempCount " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                    }
                    PageSubscriptionCounter counter = store.getCursorProvider().getSubscription(entry.getKey()).getCounter();
                    AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());
                    AtomicLong sizeValue = sizePerQueueOnPage.get(entry.getKey());
                    if (value == null) {
                        logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
                    } else {
                        logger.debug("Replacing counter " + value.get());
                        counter.increment(txRecoverCounter, value.get(), sizeValue.get());
                    }
                }
            } else {
                // on this case the page file didn't exist, we just remove all the records since the page is already gone
                logger.debug("Page " + pageId + " didn't exist on address " + addressPageMapEntry.getKey() + ", so we are just removing records");
                for (List<PageCountPending> records : perQueue.values()) {
                    for (PageCountPending record : records) {
                        logger.debug("Removing pending page counter " + record.getID());
                        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
                        txRecoverCounter.setContainsPersistent();
                    }
                }
            }
        }
    }
    txRecoverCounter.commit();
}

Example 7 with PagedMessage

use of org.apache.activemq.artemis.core.paging.PagedMessage in project activemq-artemis by apache.

the class PrintData method printPages.

private static void printPages(DescribeJournal describeJournal, StorageManager sm, PagingManager manager, PrintStream out, boolean safe) throws Exception {
    PageCursorsInfo cursorACKs = calculateCursorsInfo(describeJournal.getRecords());
    Set<Long> pgTXs = cursorACKs.getPgTXs();
    manager.start();
    SimpleString[] stores = manager.getStoreNames();
    for (SimpleString store : stores) {
        PagingStore pgStore = manager.getPageStore(store);
        File folder = null;
        if (pgStore != null) {
            folder = pgStore.getFolder();
        }
        out.println("####################################################################################################");
        out.println("Exploring store " + store + " folder = " + folder);
        int pgid = (int) pgStore.getFirstPage();
        for (int pg = 0; pg < pgStore.getNumberOfPages(); pg++) {
            out.println("*******   Page " + pgid);
            Page page = pgStore.createPage(pgid);
            page.open();
            List<PagedMessage> msgs = page.read(sm);
            page.close();
            int msgID = 0;
            for (PagedMessage msg : msgs) {
                msg.initMessage(sm);
                if (safe) {
                    try {
                        out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ", msg=" + msg.getMessage().getClass().getSimpleName() + "(safe data, size=" + msg.getMessage().getPersistentSize() + ")");
                    } catch (Exception e) {
                        out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ", msg=" + msg.getMessage().getClass().getSimpleName() + "(safe data)");
                    }
                } else {
                    out.print("pg=" + pgid + ", msg=" + msgID + ",pgTX=" + msg.getTransactionID() + ",userMessageID=" + (msg.getMessage().getUserID() != null ? msg.getMessage().getUserID() : "") + ", msg=" + msg.getMessage());
                }
                out.print(",Queues = ");
                long[] q = msg.getQueueIDs();
                for (int i = 0; i < q.length; i++) {
                    out.print(q[i]);
                    PagePosition posCheck = new PagePositionImpl(pgid, msgID);
                    boolean acked = false;
                    Set<PagePosition> positions = cursorACKs.getCursorRecords().get(q[i]);
                    if (positions != null) {
                        acked = positions.contains(posCheck);
                    }
                    if (acked) {
                        out.print(" (ACK)");
                    }
                    if (cursorACKs.getCompletePages(q[i]).contains(Long.valueOf(pgid))) {
                        out.println(" (PG-COMPLETE)");
                    }
                    if (i + 1 < q.length) {
                        out.print(",");
                    }
                }
                if (msg.getTransactionID() >= 0 && !pgTXs.contains(msg.getTransactionID())) {
                    out.print(", **PG_TX_NOT_FOUND**");
                }
                out.println();
                msgID++;
            }
            pgid++;
        }
    }
}

Example 8 with PagedMessage

use of org.apache.activemq.artemis.core.paging.PagedMessage in project activemq-artemis by apache.

the class PageSubscriptionImpl method internalGetNext.

private PagedReference internalGetNext(final PagePosition pos) {
    PagePosition retPos = pos.nextMessage();
    PageCache cache = cursorProvider.getPageCache(pos.getPageNr());
    if (cache != null && !cache.isLive() && retPos.getMessageNr() >= cache.getNumberOfMessages()) {
        // The next message is beyond what's available at the current page, so we need to move to the next page
        cache = null;
    }
    // it will scan for the next available page
    while ((cache == null && retPos.getPageNr() <= pageStore.getCurrentWritingPage()) || (cache != null && retPos.getPageNr() <= pageStore.getCurrentWritingPage() && cache.getNumberOfMessages() == 0)) {
        retPos = moveNextPage(retPos);
        cache = cursorProvider.getPageCache(retPos.getPageNr());
    }
    if (cache == null) {
        // it will be null in the case of the current writing page
        return null;
    } else {
        PagedMessage serverMessage = cache.getMessage(retPos.getMessageNr());
        if (serverMessage != null) {
            return cursorProvider.newReference(retPos, serverMessage, this);
        } else {
            return null;
        }
    }
}

Example 9 with PagedMessage

use of org.apache.activemq.artemis.core.paging.PagedMessage in project activemq-artemis by apache.

the class PagingStoreImpl method start.

@Override
public void start() throws Exception {
    lock.writeLock().lock();
    try {
        if (running) {
            // need to be ignored
            return;
        } else {
            running = true;
            firstPageId = Integer.MAX_VALUE;
            // There are no files yet on this Storage. We will just return it empty
            if (fileFactory != null) {
                currentPageId = 0;
                if (currentPage != null) {
                    currentPage.close(false);
                }
                currentPage = null;
                List<String> files = fileFactory.listFiles("page");
                numberOfPages = files.size();
                for (String fileName : files) {
                    final int fileId = PagingStoreImpl.getPageIdFromFileName(fileName);
                    if (fileId > currentPageId) {
                        currentPageId = fileId;
                    }
                    if (fileId < firstPageId) {
                        firstPageId = fileId;
                    }
                }
                if (currentPageId != 0) {
                    currentPage = createPage(currentPageId);
                    currentPage.open();
                    List<PagedMessage> messages = currentPage.read(storageManager);
                    LivePageCache pageCache = new LivePageCacheImpl(currentPage);
                    for (PagedMessage msg : messages) {
                        pageCache.addLiveMessage(msg);
                        if (msg.getMessage().isLargeMessage()) {
                            // We have to do this since addLIveMessage will increment an extra one
                            ((LargeServerMessage) msg.getMessage()).decrementDelayDeletionCount();
                        }
                    }
                    currentPage.setLiveCache(pageCache);
                    currentPageSize.set(currentPage.getSize());
                    cursorProvider.addPageCache(pageCache);
                }
                // We will not mark it for paging if there's only a single empty file
                if (currentPage != null && !(numberOfPages == 1 && currentPage.getSize() == 0)) {
                    startPaging();
                }
            }
        }
    } finally {
        lock.writeLock().unlock();
    }
}

Example 10 with PagedMessage

use of org.apache.activemq.artemis.core.paging.PagedMessage in project activemq-artemis by apache.

the class PagingStoreImpl method page.

@Override
public boolean page(Message message, final Transaction tx, RouteContextList listCtx, final ReadLock managerLock) throws Exception {
    if (!running) {
        throw new IllegalStateException("PagingStore(" + getStoreName() + ") not initialized");
    }
    boolean full = isFull();
    if (addressFullMessagePolicy == AddressFullMessagePolicy.DROP || addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
        if (full) {
            if (!printedDropMessagesWarning) {
                printedDropMessagesWarning = true;
                ActiveMQServerLogger.LOGGER.pageStoreDropMessages(storeName, sizeInBytes.get(), maxSize);
            }
            if (message.isLargeMessage()) {
                ((LargeServerMessage) message).deleteFile();
            }
            if (addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
                throw ActiveMQMessageBundle.BUNDLE.addressIsFull(address.toString());
            }
            // Address is full, we just pretend we are paging, and drop the data
            return true;
        } else {
            return false;
        }
    } else if (addressFullMessagePolicy == AddressFullMessagePolicy.BLOCK) {
        return false;
    }
    // We need to ensure a read lock, as depage could change the paging state
    lock.readLock().lock();
    try {
        // First check done concurrently, to avoid synchronization and increase throughput
        if (!paging) {
            return false;
        }
    } finally {
        lock.readLock().unlock();
    }
    if (managerLock != null) {
        managerLock.lock();
    }
    try {
        lock.writeLock().lock();
        try {
            if (!paging) {
                return false;
            }
            message.setAddress(address);
            final long transactionID = tx == null ? -1 : tx.getID();
            PagedMessage pagedMessage = new PagedMessageImpl(message, routeQueues(tx, listCtx), transactionID);
            if (message.isLargeMessage()) {
                ((LargeServerMessage) message).setPaged();
            }
            int bytesToWrite = pagedMessage.getEncodeSize() + Page.SIZE_RECORD;
            if (currentPageSize.addAndGet(bytesToWrite) > pageSize && currentPage.getNumberOfMessages() > 0) {
                // Make sure nothing is currently validating or using currentPage
                openNewPage();
                currentPageSize.addAndGet(bytesToWrite);
            }
            if (tx != null) {
                installPageTransaction(tx, listCtx);
            }
            // the apply counter will make sure we write a record on journal
            // especially on the case for non transactional sends and paging
            // doing this will give us a possibility of recovering the page counters
            long persistentSize = pagedMessage.getPersistentSize() > 0 ? pagedMessage.getPersistentSize() : 0;
            applyPageCounters(tx, getCurrentPage(), listCtx, persistentSize);
            currentPage.write(pagedMessage);
            if (tx == null && syncNonTransactional && message.isDurable()) {
                sync();
            }
            if (logger.isTraceEnabled()) {
                logger.trace("Paging message " + pagedMessage + " on pageStore " + this.getStoreName() + " pageNr=" + currentPage.getPageId());
            }
            return true;
        } finally {
            lock.writeLock().unlock();
        }
    } finally {
        if (managerLock != null) {
            managerLock.unlock();
        }
    }
}