Examples with MappedSequentialFileFactory org.apache.activemq.artemis.core.io.mapped.MappedSequentialFileFactory used on opensource projects

Example 6 with MappedSequentialFileFactory

use of org.apache.activemq.artemis.core.io.mapped.MappedSequentialFileFactory in project activemq-artemis by apache.

the class SharedNothingReplicationTest method testReplicateFromSlowLive.

@Test
public void testReplicateFromSlowLive() throws Exception {
    // start live
    Configuration liveConfiguration = createLiveConfiguration();
    ActiveMQServer liveServer = ActiveMQServers.newActiveMQServer(liveConfiguration);
    liveServer.start();
    Wait.waitFor(() -> liveServer.isStarted());
    CoreMessagePersister.theInstance = SlowMessagePersister._getInstance();
    final CountDownLatch replicated = new CountDownLatch(1);
    ServerLocator locator = ServerLocatorImpl.newLocator("tcp://localhost:61616");
    locator.setCallTimeout(60_000L);
    locator.setConnectionTTL(60_000L);
    locator.addClusterTopologyListener(new ClusterTopologyListener() {

        @Override
        public void nodeUP(TopologyMember member, boolean last) {
            logger.infof("nodeUP fired last=%s, live=%s, backup=%s", last, member.getLive(), member.getBackup());
            if (member.getBackup() != null) {
                replicated.countDown();
            }
        }

        @Override
        public void nodeDown(long eventUID, String nodeID) {
        }
    });
    final ClientSessionFactory csf = locator.createSessionFactory();
    ClientSession sess = csf.createSession();
    sess.createQueue("slow", RoutingType.ANYCAST, "slow", true);
    sess.close();
    Executor sendMessageExecutor = Executors.newCachedThreadPool();
    // let's write some messages
    int i = 0;
    final int j = 50;
    final CountDownLatch allMessageSent = new CountDownLatch(j);
    while (i < 5) {
        sendMessageExecutor.execute(() -> {
            try {
                ClientSession session = csf.createSession(true, true);
                ClientProducer producer = session.createProducer("slow");
                ClientMessage message = session.createMessage(true);
                // this will make journal's append executor busy
                message.putLongProperty("delay", 500L);
                logger.infof("try to send a message before replicated");
                producer.send(message);
                logger.info("send message done");
                producer.close();
                session.close();
                allMessageSent.countDown();
            } catch (ActiveMQException e) {
                logger.error("send message", e);
            }
        });
        i++;
    }
    // start backup
    Configuration backupConfiguration = createBackupConfiguration();
    ActiveMQServer backupServer = ActiveMQServers.newActiveMQServer(backupConfiguration);
    backupServer.start();
    Wait.waitFor(() -> backupServer.isStarted());
    Assert.assertTrue("can not replicate in 30 seconds", replicated.await(30, TimeUnit.SECONDS));
    while (i < j) {
        sendMessageExecutor.execute(() -> {
            try {
                ClientSession session = csf.createSession(true, true);
                ClientProducer producer = session.createProducer("slow");
                ClientMessage message = session.createMessage(true);
                message.putLongProperty("delay", 0L);
                logger.infof("try to send a message after replicated");
                producer.send(message);
                logger.info("send message done");
                producer.close();
                session.close();
                allMessageSent.countDown();
            } catch (ActiveMQException e) {
                logger.error("send message", e);
            }
        });
        i++;
    }
    Assert.assertTrue("all message sent", allMessageSent.await(30, TimeUnit.SECONDS));
    csf.close();
    locator.close();
    backupServer.stop(true);
    liveServer.stop(true);
    SequentialFileFactory fileFactory;
    File liveJournalDir = brokersFolder.getRoot().toPath().resolve("live").resolve("data").resolve("journal").toFile();
    fileFactory = new MappedSequentialFileFactory(liveConfiguration.getJournalLocation(), liveConfiguration.getJournalFileSize(), false, liveConfiguration.getJournalBufferSize_NIO(), liveConfiguration.getJournalBufferTimeout_NIO(), null);
    JournalImpl liveMessageJournal = new JournalImpl(liveConfiguration.getJournalFileSize(), liveConfiguration.getJournalMinFiles(), liveConfiguration.getJournalPoolFiles(), liveConfiguration.getJournalCompactMinFiles(), liveConfiguration.getJournalCompactPercentage(), fileFactory, "activemq-data", "amq", fileFactory.getMaxIO());
    liveMessageJournal.start();
    final AtomicInteger liveJournalCounter = new AtomicInteger();
    liveMessageJournal.load(new AddRecordLoaderCallback() {

        @Override
        public void addRecord(RecordInfo info) {
            if (!(info.userRecordType == JournalRecordIds.ADD_MESSAGE_PROTOCOL)) {
            // ignore
            }
            logger.infof("got live message %d", info.id);
            liveJournalCounter.incrementAndGet();
        }
    });
    // read backup's journal
    File backupJournalDir = brokersFolder.getRoot().toPath().resolve("backup").resolve("data").resolve("journal").toFile();
    fileFactory = new MappedSequentialFileFactory(backupConfiguration.getJournalLocation(), backupConfiguration.getJournalFileSize(), false, backupConfiguration.getJournalBufferSize_NIO(), backupConfiguration.getJournalBufferTimeout_NIO(), null);
    JournalImpl backupMessageJournal = new JournalImpl(backupConfiguration.getJournalFileSize(), backupConfiguration.getJournalMinFiles(), backupConfiguration.getJournalPoolFiles(), backupConfiguration.getJournalCompactMinFiles(), backupConfiguration.getJournalCompactPercentage(), fileFactory, "activemq-data", "amq", fileFactory.getMaxIO());
    backupMessageJournal.start();
    final AtomicInteger replicationCounter = new AtomicInteger();
    backupMessageJournal.load(new AddRecordLoaderCallback() {

        @Override
        public void addRecord(RecordInfo info) {
            if (!(info.userRecordType == JournalRecordIds.ADD_MESSAGE_PROTOCOL)) {
            // ignore
            }
            logger.infof("replicated message %d", info.id);
            replicationCounter.incrementAndGet();
        }
    });
    logger.infof("expected %d messages, live=%d, backup=%d", j, liveJournalCounter.get(), replicationCounter.get());
    Assert.assertEquals("Live lost journal record", j, liveJournalCounter.get());
    Assert.assertEquals("Backup did not replicated all journal", j, replicationCounter.get());
    // if this ever happens.. you need to make sure this persister is registered instead of the CoreMessagePersister
    Assert.assertTrue("The test is not valid, slow persister stopped being used", SlowMessagePersister._getInstance().used);
}

Example 7 with MappedSequentialFileFactory

use of org.apache.activemq.artemis.core.io.mapped.MappedSequentialFileFactory in project activemq-artemis by apache.

the class JournalTptBenchmark method main.

public static void main(String[] args) throws Exception {
    final boolean useDefaultIoExecutor = true;
    final int fileSize = 10 * 1024 * 1024;
    final boolean dataSync = false;
    final Type type = Type.Mapped;
    final int tests = 10;
    final int warmup = 20_000;
    final int measurements = 100_000;
    final int msgSize = 100;
    final byte[] msgContent = new byte[msgSize];
    Arrays.fill(msgContent, (byte) 1);
    final int totalMessages = (measurements * tests + warmup);
    final File tmpDirectory = new File("./");
    // using the default configuration when the broker starts!
    final SequentialFileFactory factory;
    switch(type) {
        case Mapped:
            factory = new MappedSequentialFileFactory(tmpDirectory, fileSize, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, null).setDatasync(dataSync);
            break;
        case Nio:
            factory = new NIOSequentialFileFactory(tmpDirectory, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_NIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_NIO, 1, false, null, null).setDatasync(dataSync);
            break;
        case Aio:
            factory = new AIOSequentialFileFactory(tmpDirectory, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, 500, false, null, null).setDatasync(dataSync);
            // disable it when using directly the same buffer: ((AIOSequentialFileFactory)factory).disableBufferReuse();
            if (!LibaioContext.isLoaded()) {
                throw new IllegalStateException("lib AIO not loaded!");
            }
            break;
        default:
            throw new AssertionError("unsupported case");
    }
    int numFiles = (int) (totalMessages * factory.calculateBlockSize(msgSize)) / fileSize;
    if (numFiles < 2) {
        numFiles = 2;
    }
    ExecutorService service = null;
    final Journal journal;
    if (useDefaultIoExecutor) {
        journal = new JournalImpl(fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO());
        journal.start();
    } else {
        final ArrayList<MpscArrayQueue<Runnable>> tasks = new ArrayList<>();
        service = Executors.newSingleThreadExecutor();
        journal = new JournalImpl(() -> new ArtemisExecutor() {

            private final MpscArrayQueue<Runnable> taskQueue = new MpscArrayQueue<>(1024);

            {
                tasks.add(taskQueue);
            }

            @Override
            public void execute(Runnable command) {
                while (!taskQueue.offer(command)) {
                    LockSupport.parkNanos(1L);
                }
            }
        }, fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO(), 0);
        journal.start();
        service.execute(() -> {
            final int size = tasks.size();
            final int capacity = 1024;
            while (!Thread.currentThread().isInterrupted()) {
                for (int i = 0; i < size; i++) {
                    final MpscArrayQueue<Runnable> runnables = tasks.get(i);
                    for (int j = 0; j < capacity; j++) {
                        final Runnable task = runnables.poll();
                        if (task == null) {
                            break;
                        }
                        try {
                            task.run();
                        } catch (Throwable t) {
                            System.err.println(t);
                        }
                    }
                }
            }
        });
    }
    try {
        journal.load(new ArrayList<RecordInfo>(), null, null);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    try {
        final EncodingSupport encodingSupport = new EncodingSupport() {

            @Override
            public int getEncodeSize() {
                return msgSize;
            }

            @Override
            public void encode(ActiveMQBuffer buffer) {
                final int writerIndex = buffer.writerIndex();
                buffer.setBytes(writerIndex, msgContent);
                buffer.writerIndex(writerIndex + msgSize);
            }

            @Override
            public void decode(ActiveMQBuffer buffer) {
            }
        };
        long id = 1;
        {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, warmup);
            id += warmup;
            System.out.println("warmup:" + (measurements * 1000_000_000L) / elapsed + " ops/sec");
        }
        for (int t = 0; t < tests; t++) {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, measurements);
            System.out.println((measurements * 1000_000_000L) / elapsed + " ops/sec");
            id += warmup;
        }
    } finally {
        journal.stop();
        if (service != null) {
            service.shutdown();
        }
        final File[] fileToDeletes = tmpDirectory.listFiles();
        System.out.println("Files to deletes" + Arrays.toString(fileToDeletes));
        Stream.of(fileToDeletes).forEach(File::delete);
    }
}

Example 8 with MappedSequentialFileFactory

use of org.apache.activemq.artemis.core.io.mapped.MappedSequentialFileFactory in project activemq-artemis by apache.

the class MappedJournalImplTest method getFileFactory.

@Override
protected SequentialFileFactory getFileFactory() throws Exception {
    File file = new File(getTestDir());
    deleteDirectory(file);
    file.mkdir();
    return new MappedSequentialFileFactory(getTestDirfile(), 10 * 1024, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_NIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_NIO, null);
}

Example 9 with MappedSequentialFileFactory

use of org.apache.activemq.artemis.core.io.mapped.MappedSequentialFileFactory in project activemq-artemis by apache.

the class MappedSequentialFileFactoryTest method testInterrupts.

@Test
public void testInterrupts() throws Throwable {
    final EncodingSupport fakeEncoding = new EncodingSupport() {

        @Override
        public int getEncodeSize() {
            return 10;
        }

        @Override
        public void encode(ActiveMQBuffer buffer) {
            buffer.writeBytes(new byte[10]);
        }

        @Override
        public void decode(ActiveMQBuffer buffer) {
        }
    };
    final AtomicInteger calls = new AtomicInteger(0);
    final MappedSequentialFileFactory factory = new MappedSequentialFileFactory(new File(getTestDir()), fakeEncoding.getEncodeSize(), false, 0, 0, (code, message, file) -> {
        new Exception("shutdown").printStackTrace();
        calls.incrementAndGet();
    });
    Thread threadOpen = new Thread() {

        @Override
        public void run() {
            try {
                Thread.currentThread().interrupt();
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadOpen.start();
    threadOpen.join();
    Thread threadClose = new Thread() {

        @Override
        public void run() {
            try {
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
                file.write(fakeEncoding, true);
                Thread.currentThread().interrupt();
                file.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadClose.start();
    threadClose.join();
    Thread threadWrite = new Thread() {

        @Override
        public void run() {
            try {
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
                Thread.currentThread().interrupt();
                file.write(fakeEncoding, true);
                file.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadWrite.start();
    threadWrite.join();
    Thread threadFill = new Thread() {

        @Override
        public void run() {
            try {
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
                Thread.currentThread().interrupt();
                file.fill(fakeEncoding.getEncodeSize());
                file.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadFill.start();
    threadFill.join();
    Thread threadWriteDirect = new Thread() {

        @Override
        public void run() {
            try {
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
                ByteBuffer buffer = ByteBuffer.allocate(10);
                buffer.put(new byte[10]);
                Thread.currentThread().interrupt();
                file.writeDirect(buffer, true);
                file.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadWriteDirect.start();
    threadWriteDirect.join();
    Thread threadRead = new Thread() {

        @Override
        public void run() {
            try {
                SequentialFile file = factory.createSequentialFile("file.txt");
                file.open();
                file.write(fakeEncoding, true);
                file.position(0);
                ByteBuffer readBytes = ByteBuffer.allocate(fakeEncoding.getEncodeSize());
                Thread.currentThread().interrupt();
                file.read(readBytes);
                file.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    threadRead.start();
    threadRead.join();
    // An interrupt exception shouldn't issue a shutdown
    Assert.assertEquals(0, calls.get());
}