Examples with PagingStore org.apache.activemq.artemis.core.paging.PagingStore used on opensource projects

Example 21 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method testDLAOnLargeMessageAndPaging.

@Test
public void testDLAOnLargeMessageAndPaging() throws Exception {
    clearDataRecreateServerDirs();
    Configuration config = createDefaultInVMConfig().setThreadPoolMaxSize(5).setJournalSyncNonTransactional(false);
    Map<String, AddressSettings> settings = new HashMap<>();
    AddressSettings dla = new AddressSettings().setMaxDeliveryAttempts(5).setDeadLetterAddress(new SimpleString("DLA")).setRedeliveryDelay(0);
    settings.put(ADDRESS.toString(), dla);
    server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, settings);
    server.start();
    final int messageSize = 1024;
    ServerLocator locator = null;
    ClientSessionFactory sf = null;
    ClientSession session = null;
    try {
        locator = createInVMNonHALocator();
        locator.setBlockOnNonDurableSend(true);
        locator.setBlockOnDurableSend(true);
        sf = locator.createSessionFactory();
        session = sf.createSession(false, false, false);
        session.createQueue(ADDRESS, ADDRESS, true);
        session.createQueue("DLA", "DLA", true);
        Queue serverQueue = server.locateQueue(ADDRESS);
        Queue serverQueueDLA = server.locateQueue(SimpleString.toSimpleString("DLA"));
        PagingStore pgStoreAddress = server.getPagingManager().getPageStore(ADDRESS);
        pgStoreAddress.startPaging();
        PagingStore pgStoreDLA = server.getPagingManager().getPageStore(new SimpleString("DLA"));
        ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
        for (int i = 0; i < 100; i++) {
            log.debug("send message #" + i);
            ClientMessage message = session.createMessage(true);
            message.putStringProperty("id", "str" + i);
            message.setBodyInputStream(createFakeLargeStream(messageSize));
            producer.send(message);
            if ((i + 1) % 2 == 0) {
                session.commit();
            }
        }
        session.commit();
        session.start();
        ClientConsumer cons = session.createConsumer(ADDRESS);
        for (int msgNr = 0; msgNr < 2; msgNr++) {
            for (int i = 0; i < 5; i++) {
                ClientMessage msg = cons.receive(5000);
                assertNotNull(msg);
                msg.acknowledge();
                for (int j = 0; j < messageSize; j++) {
                    assertEquals(getSamplebyte(j), msg.getBodyBuffer().readByte());
                }
                session.rollback();
            }
            pgStoreDLA.startPaging();
        }
        for (int i = 2; i < 100; i++) {
            log.debug("Received message " + i);
            ClientMessage message = cons.receive(5000);
            assertNotNull("Message " + i + " wasn't received", message);
            message.acknowledge();
            final AtomicInteger bytesOutput = new AtomicInteger(0);
            message.setOutputStream(new OutputStream() {

                @Override
                public void write(int b) throws IOException {
                    bytesOutput.incrementAndGet();
                }
            });
            try {
                if (!message.waitOutputStreamCompletion(10000)) {
                    log.info(threadDump("dump"));
                    fail("Couldn't finish large message receiving");
                }
            } catch (Throwable e) {
                log.info("output bytes = " + bytesOutput);
                log.info(threadDump("dump"));
                fail("Couldn't finish large message receiving for id=" + message.getStringProperty("id") + " with messageID=" + message.getMessageID());
            }
        }
        assertNull(cons.receiveImmediate());
        cons.close();
        cons = session.createConsumer("DLA");
        for (int i = 0; i < 2; i++) {
            assertNotNull(cons.receive(5000));
        }
        sf.close();
        session.close();
        locator.close();
        server.stop();
        server.start();
        locator = createInVMNonHALocator();
        sf = locator.createSessionFactory();
        session = sf.createSession(false, false);
        session.start();
        cons = session.createConsumer(ADDRESS);
        for (int i = 2; i < 100; i++) {
            log.debug("Received message " + i);
            ClientMessage message = cons.receive(5000);
            assertNotNull(message);
            assertEquals("str" + i, message.getStringProperty("id"));
            message.acknowledge();
            message.setOutputStream(new OutputStream() {

                @Override
                public void write(int b) throws IOException {
                }
            });
            assertTrue(message.waitOutputStreamCompletion(5000));
        }
        assertNull(cons.receiveImmediate());
        cons.close();
        cons = session.createConsumer("DLA");
        for (int msgNr = 0; msgNr < 2; msgNr++) {
            ClientMessage msg = cons.receive(10000);
            assertNotNull(msg);
            assertEquals("str" + msgNr, msg.getStringProperty("id"));
            for (int i = 0; i < messageSize; i++) {
                assertEquals(getSamplebyte(i), msg.getBodyBuffer().readByte());
            }
            msg.acknowledge();
        }
        cons.close();
        cons = session.createConsumer(ADDRESS);
        session.commit();
        assertNull(cons.receiveImmediate());
        long timeout = System.currentTimeMillis() + 5000;
        pgStoreAddress = server.getPagingManager().getPageStore(ADDRESS);
        pgStoreAddress.getCursorProvider().getSubscription(serverQueue.getID()).cleanupEntries(false);
        pgStoreAddress.getCursorProvider().cleanup();
        while (timeout > System.currentTimeMillis() && pgStoreAddress.isPaging()) {
            Thread.sleep(50);
        }
        assertFalse(pgStoreAddress.isPaging());
        session.commit();
    } finally {
        session.close();
        sf.close();
        locator.close();
        try {
            server.stop();
        } catch (Throwable ignored) {
        }
    }
}

Example 22 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method testSyncPageTX.

@Test
public void testSyncPageTX() throws Exception {
    Configuration config = createDefaultInVMConfig();
    server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
    server.start();
    server.createQueue(PagingTest.ADDRESS, RoutingType.ANYCAST, PagingTest.ADDRESS, null, true, false);
    final CountDownLatch pageUp = new CountDownLatch(0);
    final CountDownLatch pageDone = new CountDownLatch(1);
    OperationContext ctx = new DummyOperationContext(pageUp, pageDone);
    OperationContextImpl.setContext(ctx);
    PagingManager paging = server.getPagingManager();
    PagingStore store = paging.getPageStore(ADDRESS);
    store.sync();
    assertTrue(pageUp.await(10, TimeUnit.SECONDS));
    assertTrue(pageDone.await(10, TimeUnit.SECONDS));
}

Example 23 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method internalTestMultiFilters.

public void internalTestMultiFilters(boolean browsing) throws Throwable {
    clearDataRecreateServerDirs();
    Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
    server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
    server.start();
    try {
        ServerLocator locator = createInVMNonHALocator().setBlockOnDurableSend(true);
        ClientSessionFactory sf = locator.createSessionFactory();
        ClientSession session = sf.createSession(true, true, 0);
        session.createQueue(ADDRESS.toString(), "Q1", null, true);
        PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
        ClientProducer prod = session.createProducer(ADDRESS);
        ClientMessage msg = null;
        store.startPaging();
        for (int i = 0; i < 100; i++) {
            msg = session.createMessage(true);
            msg.putStringProperty("color", "red");
            msg.putIntProperty("count", i);
            prod.send(msg);
            if (i > 0 && i % 10 == 0) {
                store.startPaging();
                store.forceAnotherPage();
            }
        }
        for (int i = 0; i < 100; i++) {
            msg = session.createMessage(true);
            msg.putStringProperty("color", "green");
            msg.putIntProperty("count", i);
            prod.send(msg);
            if (i > 0 && i % 10 == 0) {
                store.startPaging();
                store.forceAnotherPage();
            }
        }
        session.commit();
        session.close();
        session = sf.createSession(false, false, 0);
        session.start();
        ClientConsumer cons1;
        if (browsing) {
            cons1 = session.createConsumer("Q1", "color='green'", true);
        } else {
            cons1 = session.createConsumer("Q1", "color='red'", false);
        }
        for (int i = 0; i < 100; i++) {
            msg = cons1.receive(5000);
            System.out.println("Received " + msg);
            assertNotNull(msg);
            if (!browsing) {
                msg.acknowledge();
            }
        }
        session.commit();
        session.close();
    } finally {
        server.stop();
    }
}

Example 24 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PagingTest method testTwoQueuesConsumeOneRestart.

@Test
public void testTwoQueuesConsumeOneRestart() throws Exception {
    boolean persistentMessages = true;
    clearDataRecreateServerDirs();
    Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
    server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
    server.start();
    final int messageSize = 1024;
    final int numberOfMessages = 1000;
    try {
        ServerLocator locator = createInVMNonHALocator().setClientFailureCheckPeriod(120000).setConnectionTTL(5000000).setCallTimeout(120000).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
        ClientSessionFactory sf = locator.createSessionFactory();
        ClientSession session = sf.createSession(false, false, false);
        session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS.concat("=1"), null, true);
        session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS.concat("=2"), null, true);
        ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
        ClientMessage message = null;
        byte[] body = new byte[messageSize];
        for (int i = 0; i < numberOfMessages; i++) {
            message = session.createMessage(persistentMessages);
            ActiveMQBuffer bodyLocal = message.getBodyBuffer();
            bodyLocal.writeBytes(body);
            message.putIntProperty("propTest", i % 2 == 0 ? 1 : 2);
            producer.send(message);
            if (i % 1000 == 0) {
                session.commit();
            }
        }
        session.commit();
        session.start();
        session.deleteQueue(PagingTest.ADDRESS.concat("=1"));
        sf = locator.createSessionFactory();
        session = sf.createSession(false, false, false);
        session.start();
        ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS.concat("=2"));
        for (int i = 0; i < numberOfMessages; i++) {
            message = consumer.receive(5000);
            assertNotNull(message);
            message.acknowledge();
        }
        session.commit();
        assertNull(consumer.receiveImmediate());
        consumer.close();
        long timeout = System.currentTimeMillis() + 10000;
        PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
        // It's async, so need to wait a bit for it happening
        while (timeout > System.currentTimeMillis() && store.isPaging()) {
            Thread.sleep(100);
        }
        assertFalse(server.getPagingManager().getPageStore(ADDRESS).isPaging());
        server.stop();
        server.start();
        server.stop();
        server.start();
        sf.close();
        locator.close();
    } finally {
        try {
            server.stop();
        } catch (Throwable ignored) {
        }
    }
}

Example 25 with PagingStore

use of org.apache.activemq.artemis.core.paging.PagingStore in project activemq-artemis by apache.

the class PostOfficeImpl method processRoute.

@Override
public void processRoute(final Message message, final RoutingContext context, final boolean direct) throws Exception {
    final List<MessageReference> refs = new ArrayList<>();
    Transaction tx = context.getTransaction();
    Long deliveryTime = message.getScheduledDeliveryTime();
    for (Map.Entry<SimpleString, RouteContextList> entry : context.getContexListing().entrySet()) {
        PagingStore store = pagingManager.getPageStore(entry.getKey());
        if (storageManager.addToPage(store, message, context.getTransaction(), entry.getValue())) {
            if (message.isLargeMessage()) {
                confirmLargeMessageSend(tx, message);
            }
            // We need to kick delivery so the Queues may check for the cursors case they are empty
            schedulePageDelivery(tx, entry);
            continue;
        }
        for (Queue queue : entry.getValue().getNonDurableQueues()) {
            MessageReference reference = MessageReference.Factory.createReference(message, queue);
            if (deliveryTime != null) {
                reference.setScheduledDeliveryTime(deliveryTime);
            }
            refs.add(reference);
            message.incrementRefCount();
        }
        Iterator<Queue> iter = entry.getValue().getDurableQueues().iterator();
        while (iter.hasNext()) {
            Queue queue = iter.next();
            MessageReference reference = MessageReference.Factory.createReference(message, queue);
            if (context.isAlreadyAcked(context.getAddress(message), queue)) {
                reference.setAlreadyAcked();
                if (tx != null) {
                    queue.acknowledge(tx, reference);
                }
            }
            if (deliveryTime != null) {
                reference.setScheduledDeliveryTime(deliveryTime);
            }
            refs.add(reference);
            if (message.isDurable()) {
                int durableRefCount = message.incrementDurableRefCount();
                if (durableRefCount == 1) {
                    if (tx != null) {
                        storageManager.storeMessageTransactional(tx.getID(), message);
                    } else {
                        storageManager.storeMessage(message);
                    }
                    if (message.isLargeMessage()) {
                        confirmLargeMessageSend(tx, message);
                    }
                }
                if (tx != null) {
                    storageManager.storeReferenceTransactional(tx.getID(), queue.getID(), message.getMessageID());
                    tx.setContainsPersistent();
                } else {
                    storageManager.storeReference(queue.getID(), message.getMessageID(), !iter.hasNext());
                }
                if (deliveryTime > 0) {
                    if (tx != null) {
                        storageManager.updateScheduledDeliveryTimeTransactional(tx.getID(), reference);
                    } else {
                        storageManager.updateScheduledDeliveryTime(reference);
                    }
                }
            }
            message.incrementRefCount();
        }
    }
    if (tx != null) {
        tx.addOperation(new AddOperation(refs));
    } else {
        // This will use the same thread if there are no pending operations
        // avoiding a context switch on this case
        storageManager.afterCompleteOperations(new IOCallback() {

            @Override
            public void onError(final int errorCode, final String errorMessage) {
                ActiveMQServerLogger.LOGGER.ioErrorAddingReferences(errorCode, errorMessage);
            }

            @Override
            public void done() {
                addReferences(refs, direct);
            }
        });
    }
}