Examples with LargeServerMessage org.apache.activemq.artemis.core.server.LargeServerMessage used on opensource projects

Example 1 with LargeServerMessage

use of org.apache.activemq.artemis.core.server.LargeServerMessage in project activemq-artemis by apache.

the class PostOfficeImpl method route.

@Override
public RoutingStatus route(final Message message, final RoutingContext context, final boolean direct, boolean rejectDuplicates, final Binding bindingMove) throws Exception {
    RoutingStatus result;
    // Sanity check
    if (message.getRefCount() > 0) {
        throw new IllegalStateException("Message cannot be routed more than once");
    }
    setPagingStore(context.getAddress(message), message);
    AtomicBoolean startedTX = new AtomicBoolean(false);
    final SimpleString address = context.getAddress(message);
    applyExpiryDelay(message, address);
    if (!checkDuplicateID(message, context, rejectDuplicates, startedTX)) {
        return RoutingStatus.DUPLICATED_ID;
    }
    message.cleanupInternalProperties();
    Bindings bindings = addressManager.getBindingsForRoutingAddress(context.getAddress(message));
    // first check for the auto-queue creation thing
    if (bindings == null) {
    // There is no queue with this address, we will check if it needs to be created
    // if (queueCreator.create(address)) {
    // TODO: this is not working!!!!
    // reassign bindings if it was created
    // bindings = addressManager.getBindingsForRoutingAddress(address);
    // }
    }
    if (bindingMove != null) {
        bindingMove.route(message, context);
    } else if (bindings != null) {
        bindings.route(message, context);
    } else {
        // this is a debug and not warn because this could be a regular scenario on publish-subscribe queues (or topic subscriptions on JMS)
        if (logger.isDebugEnabled()) {
            logger.debug("Couldn't find any bindings for address=" + address + " on message=" + message);
        }
    }
    if (server.hasBrokerPlugins()) {
        server.callBrokerPlugins(plugin -> plugin.beforeMessageRoute(message, context, direct, rejectDuplicates));
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Message after routed=" + message);
    }
    if (context.getQueueCount() == 0) {
        // Send to DLA if appropriate
        AddressSettings addressSettings = addressSettingsRepository.getMatch(address.toString());
        boolean sendToDLA = addressSettings.isSendToDLAOnNoRoute();
        if (sendToDLA) {
            // Send to the DLA for the address
            SimpleString dlaAddress = addressSettings.getDeadLetterAddress();
            if (logger.isDebugEnabled()) {
                logger.debug("sending message to dla address = " + dlaAddress + ", message=" + message);
            }
            if (dlaAddress == null) {
                result = RoutingStatus.NO_BINDINGS;
                ActiveMQServerLogger.LOGGER.noDLA(address);
            } else {
                message.referenceOriginalMessage(message, null);
                message.setAddress(dlaAddress);
                message.reencode();
                route(message, context.getTransaction(), false);
                result = RoutingStatus.NO_BINDINGS_DLA;
            }
        } else {
            result = RoutingStatus.NO_BINDINGS;
            if (logger.isDebugEnabled()) {
                logger.debug("Message " + message + " is not going anywhere as it didn't have a binding on address:" + address);
            }
            if (message.isLargeMessage()) {
                ((LargeServerMessage) message).deleteFile();
            }
        }
    } else {
        result = RoutingStatus.OK;
        try {
            processRoute(message, context, direct);
        } catch (ActiveMQAddressFullException e) {
            if (startedTX.get()) {
                context.getTransaction().rollback();
            } else if (context.getTransaction() != null) {
                context.getTransaction().markAsRollbackOnly(e);
            }
            throw e;
        }
    }
    if (startedTX.get()) {
        context.getTransaction().commit();
    }
    if (server.hasBrokerPlugins()) {
        server.callBrokerPlugins(plugin -> plugin.afterMessageRoute(message, context, direct, rejectDuplicates, result));
    }
    return result;
}

Example 2 with LargeServerMessage

use of org.apache.activemq.artemis.core.server.LargeServerMessage in project activemq-artemis by apache.

the class AbstractJournalStorageManager method loadMessageJournal.

@Override
public JournalLoadInformation loadMessageJournal(final PostOffice postOffice, final PagingManager pagingManager, final ResourceManager resourceManager, Map<Long, QueueBindingInfo> queueInfos, final Map<SimpleString, List<Pair<byte[], Long>>> duplicateIDMap, final Set<Pair<Long, Long>> pendingLargeMessages, List<PageCountPending> pendingNonTXPageCounter, final JournalLoader journalLoader) throws Exception {
    List<RecordInfo> records = new ArrayList<>();
    List<PreparedTransactionInfo> preparedTransactions = new ArrayList<>();
    Set<PageTransactionInfo> invalidPageTransactions = null;
    Map<Long, Message> messages = new HashMap<>();
    readLock();
    try {
        JournalLoadInformation info = messageJournal.load(records, preparedTransactions, new LargeMessageTXFailureCallback(this, messages));
        ArrayList<LargeServerMessage> largeMessages = new ArrayList<>();
        Map<Long, Map<Long, AddMessageRecord>> queueMap = new HashMap<>();
        Map<Long, PageSubscription> pageSubscriptions = new HashMap<>();
        final int totalSize = records.size();
        for (int reccount = 0; reccount < totalSize; reccount++) {
            // It will show log.info only with large journals (more than 1 million records)
            if (reccount > 0 && reccount % 1000000 == 0) {
                long percent = (long) ((((double) reccount) / ((double) totalSize)) * 100f);
                ActiveMQServerLogger.LOGGER.percentLoaded(percent);
            }
            RecordInfo record = records.get(reccount);
            byte[] data = record.data;
            ActiveMQBuffer buff = ActiveMQBuffers.wrappedBuffer(data);
            byte recordType = record.getUserRecordType();
            switch(recordType) {
                case JournalRecordIds.ADD_LARGE_MESSAGE_PENDING:
                    {
                        PendingLargeMessageEncoding pending = new PendingLargeMessageEncoding();
                        pending.decode(buff);
                        if (pendingLargeMessages != null) {
                            // it could be null on tests, and we don't need anything on that case
                            pendingLargeMessages.add(new Pair<>(record.id, pending.largeMessageID));
                        }
                        break;
                    }
                case JournalRecordIds.ADD_LARGE_MESSAGE:
                    {
                        LargeServerMessage largeMessage = parseLargeMessage(messages, buff);
                        messages.put(record.id, largeMessage);
                        largeMessages.add(largeMessage);
                        break;
                    }
                case JournalRecordIds.ADD_MESSAGE:
                    {
                        throw new IllegalStateException("This is using old journal data, export your data and import at the correct version");
                    }
                case JournalRecordIds.ADD_MESSAGE_PROTOCOL:
                    {
                        Message message = MessagePersister.getInstance().decode(buff, null);
                        messages.put(record.id, message);
                        break;
                    }
                case JournalRecordIds.ADD_REF:
                    {
                        long messageID = record.id;
                        RefEncoding encoding = new RefEncoding();
                        encoding.decode(buff);
                        Map<Long, AddMessageRecord> queueMessages = queueMap.get(encoding.queueID);
                        if (queueMessages == null) {
                            queueMessages = new LinkedHashMap<>();
                            queueMap.put(encoding.queueID, queueMessages);
                        }
                        Message message = messages.get(messageID);
                        if (message == null) {
                            ActiveMQServerLogger.LOGGER.cannotFindMessage(record.id);
                        } else {
                            queueMessages.put(messageID, new AddMessageRecord(message));
                        }
                        break;
                    }
                case JournalRecordIds.ACKNOWLEDGE_REF:
                    {
                        long messageID = record.id;
                        RefEncoding encoding = new RefEncoding();
                        encoding.decode(buff);
                        Map<Long, AddMessageRecord> queueMessages = queueMap.get(encoding.queueID);
                        if (queueMessages == null) {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueue(encoding.queueID, messageID);
                        } else {
                            AddMessageRecord rec = queueMessages.remove(messageID);
                            if (rec == null) {
                                ActiveMQServerLogger.LOGGER.cannotFindMessage(messageID);
                            }
                        }
                        break;
                    }
                case JournalRecordIds.UPDATE_DELIVERY_COUNT:
                    {
                        long messageID = record.id;
                        DeliveryCountUpdateEncoding encoding = new DeliveryCountUpdateEncoding();
                        encoding.decode(buff);
                        Map<Long, AddMessageRecord> queueMessages = queueMap.get(encoding.queueID);
                        if (queueMessages == null) {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueueDelCount(encoding.queueID);
                        } else {
                            AddMessageRecord rec = queueMessages.get(messageID);
                            if (rec == null) {
                                ActiveMQServerLogger.LOGGER.journalCannotFindMessageDelCount(messageID);
                            } else {
                                rec.setDeliveryCount(encoding.count);
                            }
                        }
                        break;
                    }
                case JournalRecordIds.PAGE_TRANSACTION:
                    {
                        PageTransactionInfo invalidPGTx = null;
                        if (record.isUpdate) {
                            PageUpdateTXEncoding pageUpdate = new PageUpdateTXEncoding();
                            pageUpdate.decode(buff);
                            PageTransactionInfo pageTX = pagingManager.getTransaction(pageUpdate.pageTX);
                            if (pageTX == null) {
                                ActiveMQServerLogger.LOGGER.journalCannotFindPageTX(pageUpdate.pageTX);
                            } else {
                                if (!pageTX.onUpdate(pageUpdate.recods, null, null)) {
                                    invalidPGTx = pageTX;
                                }
                            }
                        } else {
                            PageTransactionInfoImpl pageTransactionInfo = new PageTransactionInfoImpl();
                            pageTransactionInfo.decode(buff);
                            pageTransactionInfo.setRecordID(record.id);
                            pagingManager.addTransaction(pageTransactionInfo);
                            if (!pageTransactionInfo.checkSize(null, null)) {
                                invalidPGTx = pageTransactionInfo;
                            }
                        }
                        if (invalidPGTx != null) {
                            if (invalidPageTransactions == null) {
                                invalidPageTransactions = new HashSet<>();
                            }
                            invalidPageTransactions.add(invalidPGTx);
                        }
                        break;
                    }
                case JournalRecordIds.SET_SCHEDULED_DELIVERY_TIME:
                    {
                        long messageID = record.id;
                        ScheduledDeliveryEncoding encoding = new ScheduledDeliveryEncoding();
                        encoding.decode(buff);
                        Map<Long, AddMessageRecord> queueMessages = queueMap.get(encoding.queueID);
                        if (queueMessages == null) {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueueScheduled(encoding.queueID, messageID);
                        } else {
                            AddMessageRecord rec = queueMessages.get(messageID);
                            if (rec == null) {
                                ActiveMQServerLogger.LOGGER.cannotFindMessage(messageID);
                            } else {
                                rec.setScheduledDeliveryTime(encoding.scheduledDeliveryTime);
                            }
                        }
                        break;
                    }
                case JournalRecordIds.DUPLICATE_ID:
                    {
                        DuplicateIDEncoding encoding = new DuplicateIDEncoding();
                        encoding.decode(buff);
                        List<Pair<byte[], Long>> ids = duplicateIDMap.get(encoding.address);
                        if (ids == null) {
                            ids = new ArrayList<>();
                            duplicateIDMap.put(encoding.address, ids);
                        }
                        ids.add(new Pair<>(encoding.duplID, record.id));
                        break;
                    }
                case JournalRecordIds.HEURISTIC_COMPLETION:
                    {
                        HeuristicCompletionEncoding encoding = new HeuristicCompletionEncoding();
                        encoding.decode(buff);
                        resourceManager.putHeuristicCompletion(record.id, encoding.xid, encoding.isCommit);
                        break;
                    }
                case JournalRecordIds.ACKNOWLEDGE_CURSOR:
                    {
                        CursorAckRecordEncoding encoding = new CursorAckRecordEncoding();
                        encoding.decode(buff);
                        encoding.position.setRecordID(record.id);
                        PageSubscription sub = locateSubscription(encoding.queueID, pageSubscriptions, queueInfos, pagingManager);
                        if (sub != null) {
                            sub.reloadACK(encoding.position);
                        } else {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueueReloading(encoding.queueID);
                            messageJournal.appendDeleteRecord(record.id, false);
                        }
                        break;
                    }
                case JournalRecordIds.PAGE_CURSOR_COUNTER_VALUE:
                    {
                        PageCountRecord encoding = new PageCountRecord();
                        encoding.decode(buff);
                        PageSubscription sub = locateSubscription(encoding.getQueueID(), pageSubscriptions, queueInfos, pagingManager);
                        if (sub != null) {
                            sub.getCounter().loadValue(record.id, encoding.getValue(), encoding.getPersistentSize());
                        } else {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueueReloadingPage(encoding.getQueueID());
                            messageJournal.appendDeleteRecord(record.id, false);
                        }
                        break;
                    }
                case JournalRecordIds.PAGE_CURSOR_COUNTER_INC:
                    {
                        PageCountRecordInc encoding = new PageCountRecordInc();
                        encoding.decode(buff);
                        PageSubscription sub = locateSubscription(encoding.getQueueID(), pageSubscriptions, queueInfos, pagingManager);
                        if (sub != null) {
                            sub.getCounter().loadInc(record.id, encoding.getValue(), encoding.getPersistentSize());
                        } else {
                            ActiveMQServerLogger.LOGGER.journalCannotFindQueueReloadingPageCursor(encoding.getQueueID());
                            messageJournal.appendDeleteRecord(record.id, false);
                        }
                        break;
                    }
                case JournalRecordIds.PAGE_CURSOR_COMPLETE:
                    {
                        CursorAckRecordEncoding encoding = new CursorAckRecordEncoding();
                        encoding.decode(buff);
                        encoding.position.setRecordID(record.id);
                        PageSubscription sub = locateSubscription(encoding.queueID, pageSubscriptions, queueInfos, pagingManager);
                        if (sub != null) {
                            if (!sub.reloadPageCompletion(encoding.position)) {
                                if (logger.isDebugEnabled()) {
                                    logger.debug("Complete page " + encoding.position.getPageNr() + " doesn't exist on page manager " + sub.getPagingStore().getAddress());
                                }
                                messageJournal.appendDeleteRecord(record.id, false);
                            }
                        } else {
                            ActiveMQServerLogger.LOGGER.cantFindQueueOnPageComplete(encoding.queueID);
                            messageJournal.appendDeleteRecord(record.id, false);
                        }
                        break;
                    }
                case JournalRecordIds.PAGE_CURSOR_PENDING_COUNTER:
                    {
                        PageCountPendingImpl pendingCountEncoding = new PageCountPendingImpl();
                        pendingCountEncoding.decode(buff);
                        pendingCountEncoding.setID(record.id);
                        // This can be null on testcases not interested on this outcome
                        if (pendingNonTXPageCounter != null) {
                            pendingNonTXPageCounter.add(pendingCountEncoding);
                        }
                        break;
                    }
                default:
                    {
                        throw new IllegalStateException("Invalid record type " + recordType);
                    }
            }
            // This will free up memory sooner. The record is not needed any more
            // and its byte array would consume memory during the load process even though it's not necessary any longer
            // what would delay processing time during load
            records.set(reccount, null);
        }
        // Release the memory as soon as not needed any longer
        records.clear();
        records = null;
        journalLoader.handleAddMessage(queueMap);
        loadPreparedTransactions(postOffice, pagingManager, resourceManager, queueInfos, preparedTransactions, duplicateIDMap, pageSubscriptions, pendingLargeMessages, journalLoader);
        for (PageSubscription sub : pageSubscriptions.values()) {
            sub.getCounter().processReload();
        }
        for (LargeServerMessage msg : largeMessages) {
            if (msg.getRefCount() == 0) {
                ActiveMQServerLogger.LOGGER.largeMessageWithNoRef(msg.getMessageID());
                msg.decrementDelayDeletionCount();
            }
        }
        journalLoader.handleNoMessageReferences(messages);
        // To recover positions on Iterators
        if (pagingManager != null) {
            // it could be null on certain tests that are not dealing with paging
            // This could also be the case in certain embedded conditions
            pagingManager.processReload();
        }
        journalLoader.postLoad(messageJournal, resourceManager, duplicateIDMap);
        checkInvalidPageTransactions(pagingManager, invalidPageTransactions);
        journalLoaded = true;
        return info;
    } finally {
        readUnLock();
    }
}

Example 3 with LargeServerMessage

use of org.apache.activemq.artemis.core.server.LargeServerMessage in project activemq-artemis by apache.

the class DescribeJournal method newObjectEncoding.

public static Object newObjectEncoding(RecordInfo info, JournalStorageManager storageManager) {
    ActiveMQBuffer buffer = ActiveMQBuffers.wrappedBuffer(info.data);
    long id = info.id;
    int rec = info.getUserRecordType();
    switch(rec) {
        case ADD_LARGE_MESSAGE_PENDING:
            {
                PendingLargeMessageEncoding lmEncoding = new PendingLargeMessageEncoding();
                lmEncoding.decode(buffer);
                return lmEncoding;
            }
        case ADD_LARGE_MESSAGE:
            {
                LargeServerMessage largeMessage = new LargeServerMessageImpl(storageManager);
                LargeMessagePersister.getInstance().decode(buffer, largeMessage);
                return new MessageDescribe(largeMessage);
            }
        case ADD_MESSAGE:
            {
                return "ADD-MESSAGE is not supported any longer, use export/import";
            }
        case ADD_MESSAGE_PROTOCOL:
            {
                Message message = MessagePersister.getInstance().decode(buffer, null);
                return new MessageDescribe(message);
            }
        case ADD_REF:
            {
                final RefEncoding encoding = new RefEncoding();
                encoding.decode(buffer);
                return new ReferenceDescribe(encoding);
            }
        case ACKNOWLEDGE_REF:
            {
                final RefEncoding encoding = new RefEncoding();
                encoding.decode(buffer);
                return new AckDescribe(encoding);
            }
        case UPDATE_DELIVERY_COUNT:
            {
                DeliveryCountUpdateEncoding updateDeliveryCount = new DeliveryCountUpdateEncoding();
                updateDeliveryCount.decode(buffer);
                return updateDeliveryCount;
            }
        case PAGE_TRANSACTION:
            {
                if (info.isUpdate) {
                    PageUpdateTXEncoding pageUpdate = new PageUpdateTXEncoding();
                    pageUpdate.decode(buffer);
                    return pageUpdate;
                } else {
                    PageTransactionInfoImpl pageTransactionInfo = new PageTransactionInfoImpl();
                    pageTransactionInfo.decode(buffer);
                    pageTransactionInfo.setRecordID(info.id);
                    return pageTransactionInfo;
                }
            }
        case SET_SCHEDULED_DELIVERY_TIME:
            {
                ScheduledDeliveryEncoding encoding = new ScheduledDeliveryEncoding();
                encoding.decode(buffer);
                return encoding;
            }
        case DUPLICATE_ID:
            {
                DuplicateIDEncoding encoding = new DuplicateIDEncoding();
                encoding.decode(buffer);
                return encoding;
            }
        case HEURISTIC_COMPLETION:
            {
                HeuristicCompletionEncoding encoding = new HeuristicCompletionEncoding();
                encoding.decode(buffer);
                return encoding;
            }
        case ACKNOWLEDGE_CURSOR:
            {
                CursorAckRecordEncoding encoding = new CursorAckRecordEncoding();
                encoding.decode(buffer);
                return encoding;
            }
        case PAGE_CURSOR_COUNTER_VALUE:
            {
                PageCountRecord encoding = new PageCountRecord();
                encoding.decode(buffer);
                return encoding;
            }
        case PAGE_CURSOR_COMPLETE:
            {
                CursorAckRecordEncoding encoding = new PageCompleteCursorAckRecordEncoding();
                encoding.decode(buffer);
                return encoding;
            }
        case PAGE_CURSOR_COUNTER_INC:
            {
                PageCountRecordInc encoding = new PageCountRecordInc();
                encoding.decode(buffer);
                return encoding;
            }
        case PAGE_CURSOR_PENDING_COUNTER:
            {
                PageCountPendingImpl encoding = new PageCountPendingImpl();
                encoding.decode(buffer);
                encoding.setID(info.id);
                return encoding;
            }
        case QUEUE_STATUS_RECORD:
            return AbstractJournalStorageManager.newQueueStatusEncoding(id, buffer);
        case QUEUE_BINDING_RECORD:
            return AbstractJournalStorageManager.newQueueBindingEncoding(id, buffer);
        case ID_COUNTER_RECORD:
            EncodingSupport idReturn = new IDCounterEncoding();
            idReturn.decode(buffer);
            return idReturn;
        case JournalRecordIds.GROUP_RECORD:
            return AbstractJournalStorageManager.newGroupEncoding(id, buffer);
        case ADDRESS_SETTING_RECORD:
            return AbstractJournalStorageManager.newAddressEncoding(id, buffer);
        case SECURITY_RECORD:
            return AbstractJournalStorageManager.newSecurityRecord(id, buffer);
        case ADDRESS_BINDING_RECORD:
            return AbstractJournalStorageManager.newAddressBindingEncoding(id, buffer);
        default:
            return null;
    }
}

Example 4 with LargeServerMessage

use of org.apache.activemq.artemis.core.server.LargeServerMessage in project activemq-artemis by apache.

the class LargeServerMessageImpl method copy.

@Override
public Message copy(final long newID) {
    try {
        LargeServerMessage newMessage = storageManager.createLargeMessage(newID, this);
        boolean originallyOpen = file != null && file.isOpen();
        validateFile();
        byte[] bufferBytes = new byte[100 * 1024];
        ByteBuffer buffer = ByteBuffer.wrap(bufferBytes);
        long oldPosition = file.position();
        file.open();
        file.position(0);
        for (; ; ) {
            // The buffer is reused...
            // We need to make sure we clear the limits and the buffer before reusing it
            buffer.clear();
            int bytesRead = file.read(buffer);
            byte[] bufferToWrite;
            if (bytesRead <= 0) {
                break;
            } else if (bytesRead == bufferBytes.length && !this.storageManager.isReplicated()) {
                // ARTEMIS-1220: We cannot reuse the same buffer if it's replicated
                // otherwise there could be another thread still using the buffer on a
                // replication.
                bufferToWrite = bufferBytes;
            } else {
                bufferToWrite = new byte[bytesRead];
                System.arraycopy(bufferBytes, 0, bufferToWrite, 0, bytesRead);
            }
            newMessage.addBytes(bufferToWrite);
            if (bytesRead < bufferBytes.length) {
                break;
            }
        }
        file.position(oldPosition);
        if (!originallyOpen) {
            file.close();
            newMessage.getFile().close();
        }
        return newMessage;
    } catch (Exception e) {
        ActiveMQServerLogger.LOGGER.lareMessageErrorCopying(e, this);
        return null;
    }
}

Example 5 with LargeServerMessage

use of org.apache.activemq.artemis.core.server.LargeServerMessage in project activemq-artemis by apache.

the class LargeMessageTXFailureCallback method failedTransaction.

@Override
public void failedTransaction(final long transactionID, final List<RecordInfo> records, final List<RecordInfo> recordsToDelete) {
    for (RecordInfo record : records) {
        if (record.userRecordType == ADD_LARGE_MESSAGE) {
            byte[] data = record.data;
            ActiveMQBuffer buff = ActiveMQBuffers.wrappedBuffer(data);
            try {
                LargeServerMessage serverMessage = journalStorageManager.parseLargeMessage(messages, buff);
                serverMessage.decrementDelayDeletionCount();
            } catch (Exception e) {
                ActiveMQServerLogger.LOGGER.journalError(e);
            }
        }
    }
}