Examples with JournalImpl org.apache.activemq.artemis.core.journal.impl.JournalImpl used on opensource projects

Example 26 with JournalImpl

use of org.apache.activemq.artemis.core.journal.impl.JournalImpl in project activemq-artemis by apache.

the class BackupSyncJournalTest method testReserveFileIdValuesOnBackup.

@Test
public void testReserveFileIdValuesOnBackup() throws Exception {
    final int totalRounds = 5;
    createProducerSendSomeMessages();
    JournalImpl messageJournal = getMessageJournalFromServer(liveServer);
    for (int i = 0; i < totalRounds; i++) {
        messageJournal.forceMoveNextFile();
        sendMessages(session, producer, n_msgs);
    }
    Queue queue = liveServer.getServer().locateQueue(ADDRESS);
    PagingStore store = queue.getPageSubscription().getPagingStore();
    // what would make the verification on similar journal to fail after the recovery
    if (store.isPaging()) {
        store.forceAnotherPage();
    }
    backupServer.start();
    // Deliver messages with Backup in-sync
    waitForRemoteBackup(sessionFactory, BACKUP_WAIT_TIME, false, backupServer.getServer());
    final JournalImpl backupMsgJournal = getMessageJournalFromServer(backupServer);
    sendMessages(session, producer, n_msgs);
    // what would make the verification on similar journal to fail after the recovery
    if (store.isPaging()) {
        store.forceAnotherPage();
    }
    // Deliver messages with Backup up-to-date
    syncDelay.deliverUpToDateMsg();
    waitForRemoteBackup(sessionFactory, BACKUP_WAIT_TIME, true, backupServer.getServer());
    // SEND more messages, now with the backup replicating
    sendMessages(session, producer, n_msgs);
    // what would make the verification on similar journal to fail after the recovery
    if (store.isPaging()) {
        store.forceAnotherPage();
    }
    Set<Pair<Long, Integer>> liveIds = getFileIds(messageJournal);
    int size = messageJournal.getFileSize();
    PagingStore ps = liveServer.getServer().getPagingManager().getPageStore(ADDRESS);
    if (ps.getPageSizeBytes() == PAGE_SIZE) {
        assertTrue("isStarted", ps.isStarted());
        assertFalse("start paging should return false, because we expect paging to be running", ps.startPaging());
    }
    finishSyncAndFailover();
    assertEquals("file sizes must be the same", size, backupMsgJournal.getFileSize());
    Set<Pair<Long, Integer>> backupIds = getFileIds(backupMsgJournal);
    int total = 0;
    for (Pair<Long, Integer> pair : liveIds) {
        total += pair.getB();
    }
    int totalBackup = 0;
    for (Pair<Long, Integer> pair : backupIds) {
        totalBackup += pair.getB();
    }
    assertEquals("number of records must match ", total, totalBackup);
    // "+ 2": there two other calls that send N_MSGS.
    for (int i = 0; i < totalRounds + 3; i++) {
        receiveMsgsInRange(0, n_msgs);
    }
    assertNoMoreMessages();
}

Example 27 with JournalImpl

use of org.apache.activemq.artemis.core.journal.impl.JournalImpl in project activemq-artemis by apache.

the class SharedNothingReplicationTest method testReplicateFromSlowLive.

@Test
public void testReplicateFromSlowLive() throws Exception {
    // start live
    Configuration liveConfiguration = createLiveConfiguration();
    ActiveMQServer liveServer = ActiveMQServers.newActiveMQServer(liveConfiguration);
    liveServer.start();
    Wait.waitFor(() -> liveServer.isStarted());
    CoreMessagePersister.theInstance = SlowMessagePersister._getInstance();
    final CountDownLatch replicated = new CountDownLatch(1);
    ServerLocator locator = ServerLocatorImpl.newLocator("tcp://localhost:61616");
    locator.setCallTimeout(60_000L);
    locator.setConnectionTTL(60_000L);
    locator.addClusterTopologyListener(new ClusterTopologyListener() {

        @Override
        public void nodeUP(TopologyMember member, boolean last) {
            logger.infof("nodeUP fired last=%s, live=%s, backup=%s", last, member.getLive(), member.getBackup());
            if (member.getBackup() != null) {
                replicated.countDown();
            }
        }

        @Override
        public void nodeDown(long eventUID, String nodeID) {
        }
    });
    final ClientSessionFactory csf = locator.createSessionFactory();
    ClientSession sess = csf.createSession();
    sess.createQueue("slow", RoutingType.ANYCAST, "slow", true);
    sess.close();
    Executor sendMessageExecutor = Executors.newCachedThreadPool();
    // let's write some messages
    int i = 0;
    final int j = 50;
    final CountDownLatch allMessageSent = new CountDownLatch(j);
    while (i < 5) {
        sendMessageExecutor.execute(() -> {
            try {
                ClientSession session = csf.createSession(true, true);
                ClientProducer producer = session.createProducer("slow");
                ClientMessage message = session.createMessage(true);
                // this will make journal's append executor busy
                message.putLongProperty("delay", 500L);
                logger.infof("try to send a message before replicated");
                producer.send(message);
                logger.info("send message done");
                producer.close();
                session.close();
                allMessageSent.countDown();
            } catch (ActiveMQException e) {
                logger.error("send message", e);
            }
        });
        i++;
    }
    // start backup
    Configuration backupConfiguration = createBackupConfiguration();
    ActiveMQServer backupServer = ActiveMQServers.newActiveMQServer(backupConfiguration);
    backupServer.start();
    Wait.waitFor(() -> backupServer.isStarted());
    Assert.assertTrue("can not replicate in 30 seconds", replicated.await(30, TimeUnit.SECONDS));
    while (i < j) {
        sendMessageExecutor.execute(() -> {
            try {
                ClientSession session = csf.createSession(true, true);
                ClientProducer producer = session.createProducer("slow");
                ClientMessage message = session.createMessage(true);
                message.putLongProperty("delay", 0L);
                logger.infof("try to send a message after replicated");
                producer.send(message);
                logger.info("send message done");
                producer.close();
                session.close();
                allMessageSent.countDown();
            } catch (ActiveMQException e) {
                logger.error("send message", e);
            }
        });
        i++;
    }
    Assert.assertTrue("all message sent", allMessageSent.await(30, TimeUnit.SECONDS));
    csf.close();
    locator.close();
    backupServer.stop(true);
    liveServer.stop(true);
    SequentialFileFactory fileFactory;
    File liveJournalDir = brokersFolder.getRoot().toPath().resolve("live").resolve("data").resolve("journal").toFile();
    fileFactory = new MappedSequentialFileFactory(liveConfiguration.getJournalLocation(), liveConfiguration.getJournalFileSize(), false, liveConfiguration.getJournalBufferSize_NIO(), liveConfiguration.getJournalBufferTimeout_NIO(), null);
    JournalImpl liveMessageJournal = new JournalImpl(liveConfiguration.getJournalFileSize(), liveConfiguration.getJournalMinFiles(), liveConfiguration.getJournalPoolFiles(), liveConfiguration.getJournalCompactMinFiles(), liveConfiguration.getJournalCompactPercentage(), fileFactory, "activemq-data", "amq", fileFactory.getMaxIO());
    liveMessageJournal.start();
    final AtomicInteger liveJournalCounter = new AtomicInteger();
    liveMessageJournal.load(new AddRecordLoaderCallback() {

        @Override
        public void addRecord(RecordInfo info) {
            if (!(info.userRecordType == JournalRecordIds.ADD_MESSAGE_PROTOCOL)) {
            // ignore
            }
            logger.infof("got live message %d", info.id);
            liveJournalCounter.incrementAndGet();
        }
    });
    // read backup's journal
    File backupJournalDir = brokersFolder.getRoot().toPath().resolve("backup").resolve("data").resolve("journal").toFile();
    fileFactory = new MappedSequentialFileFactory(backupConfiguration.getJournalLocation(), backupConfiguration.getJournalFileSize(), false, backupConfiguration.getJournalBufferSize_NIO(), backupConfiguration.getJournalBufferTimeout_NIO(), null);
    JournalImpl backupMessageJournal = new JournalImpl(backupConfiguration.getJournalFileSize(), backupConfiguration.getJournalMinFiles(), backupConfiguration.getJournalPoolFiles(), backupConfiguration.getJournalCompactMinFiles(), backupConfiguration.getJournalCompactPercentage(), fileFactory, "activemq-data", "amq", fileFactory.getMaxIO());
    backupMessageJournal.start();
    final AtomicInteger replicationCounter = new AtomicInteger();
    backupMessageJournal.load(new AddRecordLoaderCallback() {

        @Override
        public void addRecord(RecordInfo info) {
            if (!(info.userRecordType == JournalRecordIds.ADD_MESSAGE_PROTOCOL)) {
            // ignore
            }
            logger.infof("replicated message %d", info.id);
            replicationCounter.incrementAndGet();
        }
    });
    logger.infof("expected %d messages, live=%d, backup=%d", j, liveJournalCounter.get(), replicationCounter.get());
    Assert.assertEquals("Live lost journal record", j, liveJournalCounter.get());
    Assert.assertEquals("Backup did not replicated all journal", j, replicationCounter.get());
    // if this ever happens.. you need to make sure this persister is registered instead of the CoreMessagePersister
    Assert.assertTrue("The test is not valid, slow persister stopped being used", SlowMessagePersister._getInstance().used);
}

Example 28 with JournalImpl

use of org.apache.activemq.artemis.core.journal.impl.JournalImpl in project activemq-artemis by apache.

the class CompactJournal method compactJournal.

private void compactJournal(final File directory, final String journalPrefix, final String journalSuffix, final int minFiles, final int fileSize, final IOCriticalErrorListener listener) throws Exception {
    NIOSequentialFileFactory nio = new NIOSequentialFileFactory(directory, listener, 1);
    JournalImpl journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, nio, journalPrefix, journalSuffix, 1);
    journal.start();
    journal.loadInternalOnly();
    journal.compact();
    journal.stop();
}

Example 29 with JournalImpl

use of org.apache.activemq.artemis.core.journal.impl.JournalImpl in project activemq-artemis by apache.

the class EncodeJournal method exportJournal.

public static void exportJournal(final String directory, final String journalPrefix, final String journalSuffix, final int minFiles, final int fileSize, final PrintStream out) throws Exception {
    NIOSequentialFileFactory nio = new NIOSequentialFileFactory(new File(directory), null, 1);
    JournalImpl journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, nio, journalPrefix, journalSuffix, 1);
    List<JournalFile> files = journal.orderFiles();
    for (JournalFile file : files) {
        out.println("#File," + file);
        exportJournalFile(out, nio, file);
    }
}

Example 30 with JournalImpl

use of org.apache.activemq.artemis.core.journal.impl.JournalImpl in project activemq-artemis by apache.

the class JournalTptBenchmark method main.

public static void main(String[] args) throws Exception {
    final boolean useDefaultIoExecutor = true;
    final int fileSize = 10 * 1024 * 1024;
    final boolean dataSync = false;
    final Type type = Type.Mapped;
    final int tests = 10;
    final int warmup = 20_000;
    final int measurements = 100_000;
    final int msgSize = 100;
    final byte[] msgContent = new byte[msgSize];
    Arrays.fill(msgContent, (byte) 1);
    final int totalMessages = (measurements * tests + warmup);
    final File tmpDirectory = new File("./");
    // using the default configuration when the broker starts!
    final SequentialFileFactory factory;
    switch(type) {
        case Mapped:
            factory = new MappedSequentialFileFactory(tmpDirectory, fileSize, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, null).setDatasync(dataSync);
            break;
        case Nio:
            factory = new NIOSequentialFileFactory(tmpDirectory, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_NIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_NIO, 1, false, null, null).setDatasync(dataSync);
            break;
        case Aio:
            factory = new AIOSequentialFileFactory(tmpDirectory, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, 500, false, null, null).setDatasync(dataSync);
            // disable it when using directly the same buffer: ((AIOSequentialFileFactory)factory).disableBufferReuse();
            if (!LibaioContext.isLoaded()) {
                throw new IllegalStateException("lib AIO not loaded!");
            }
            break;
        default:
            throw new AssertionError("unsupported case");
    }
    int numFiles = (int) (totalMessages * factory.calculateBlockSize(msgSize)) / fileSize;
    if (numFiles < 2) {
        numFiles = 2;
    }
    ExecutorService service = null;
    final Journal journal;
    if (useDefaultIoExecutor) {
        journal = new JournalImpl(fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO());
        journal.start();
    } else {
        final ArrayList<MpscArrayQueue<Runnable>> tasks = new ArrayList<>();
        service = Executors.newSingleThreadExecutor();
        journal = new JournalImpl(() -> new ArtemisExecutor() {

            private final MpscArrayQueue<Runnable> taskQueue = new MpscArrayQueue<>(1024);

            {
                tasks.add(taskQueue);
            }

            @Override
            public void execute(Runnable command) {
                while (!taskQueue.offer(command)) {
                    LockSupport.parkNanos(1L);
                }
            }
        }, fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO(), 0);
        journal.start();
        service.execute(() -> {
            final int size = tasks.size();
            final int capacity = 1024;
            while (!Thread.currentThread().isInterrupted()) {
                for (int i = 0; i < size; i++) {
                    final MpscArrayQueue<Runnable> runnables = tasks.get(i);
                    for (int j = 0; j < capacity; j++) {
                        final Runnable task = runnables.poll();
                        if (task == null) {
                            break;
                        }
                        try {
                            task.run();
                        } catch (Throwable t) {
                            System.err.println(t);
                        }
                    }
                }
            }
        });
    }
    try {
        journal.load(new ArrayList<RecordInfo>(), null, null);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    try {
        final EncodingSupport encodingSupport = new EncodingSupport() {

            @Override
            public int getEncodeSize() {
                return msgSize;
            }

            @Override
            public void encode(ActiveMQBuffer buffer) {
                final int writerIndex = buffer.writerIndex();
                buffer.setBytes(writerIndex, msgContent);
                buffer.writerIndex(writerIndex + msgSize);
            }

            @Override
            public void decode(ActiveMQBuffer buffer) {
            }
        };
        long id = 1;
        {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, warmup);
            id += warmup;
            System.out.println("warmup:" + (measurements * 1000_000_000L) / elapsed + " ops/sec");
        }
        for (int t = 0; t < tests; t++) {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, measurements);
            System.out.println((measurements * 1000_000_000L) / elapsed + " ops/sec");
            id += warmup;
        }
    } finally {
        journal.stop();
        if (service != null) {
            service.shutdown();
        }
        final File[] fileToDeletes = tmpDirectory.listFiles();
        System.out.println("Files to deletes" + Arrays.toString(fileToDeletes));
        Stream.of(fileToDeletes).forEach(File::delete);
    }
}