Examples with RecordInfo org.apache.activemq.artemis.core.journal.RecordInfo used on opensource projects

Example 51 with RecordInfo

use of org.apache.activemq.artemis.core.journal.RecordInfo in project activemq-artemis by apache.

the class AbstractJournalUpdateTask method readControlFile.

public static SequentialFile readControlFile(final SequentialFileFactory fileFactory, final List<String> dataFiles, final List<String> newFiles, final List<Pair<String, String>> renameFile) throws Exception {
    SequentialFile controlFile = fileFactory.createSequentialFile(AbstractJournalUpdateTask.FILE_COMPACT_CONTROL);
    if (controlFile.exists()) {
        JournalFile file = new JournalFileImpl(controlFile, 0, JournalImpl.FORMAT_VERSION);
        final ArrayList<RecordInfo> records = new ArrayList<>();
        JournalImpl.readJournalFile(fileFactory, file, new JournalReaderCallbackAbstract() {

            @Override
            public void onReadAddRecord(final RecordInfo info) throws Exception {
                records.add(info);
            }
        });
        if (records.size() == 0) {
            // the record is damaged
            controlFile.delete();
            return null;
        } else {
            ActiveMQBuffer input = ActiveMQBuffers.wrappedBuffer(records.get(0).data);
            int numberDataFiles = input.readInt();
            for (int i = 0; i < numberDataFiles; i++) {
                dataFiles.add(input.readUTF());
            }
            int numberNewFiles = input.readInt();
            for (int i = 0; i < numberNewFiles; i++) {
                newFiles.add(input.readUTF());
            }
            int numberRenames = input.readInt();
            for (int i = 0; i < numberRenames; i++) {
                String from = input.readUTF();
                String to = input.readUTF();
                renameFile.add(new Pair<>(from, to));
            }
        }
        return controlFile;
    } else {
        return null;
    }
}

Example 52 with RecordInfo

use of org.apache.activemq.artemis.core.journal.RecordInfo in project activemq-artemis by apache.

the class JournalImpl method load.

/**
 * @see JournalImpl#load(LoaderCallback)
 */
@Override
public synchronized JournalLoadInformation load(final List<RecordInfo> committedRecords, final List<PreparedTransactionInfo> preparedTransactions, final TransactionFailureCallback failureCallback, final boolean fixBadTX) throws Exception {
    final Set<Long> recordsToDelete = new HashSet<>();
    // ArrayList was taking too long to delete elements on checkDeleteSize
    final List<RecordInfo> records = new LinkedList<>();
    final int DELETE_FLUSH = 20000;
    JournalLoadInformation info = load(new LoaderCallback() {

        Runtime runtime = Runtime.getRuntime();

        private void checkDeleteSize() {
            // HORNETQ-482 - Flush deletes only if memory is critical
            if (recordsToDelete.size() > DELETE_FLUSH && runtime.freeMemory() < runtime.maxMemory() * 0.2) {
                ActiveMQJournalLogger.LOGGER.debug("Flushing deletes during loading, deleteCount = " + recordsToDelete.size());
                // Clean up when the list is too large, or it won't be possible to load large sets of files
                // Done as part of JBMESSAGING-1678
                Iterator<RecordInfo> iter = records.iterator();
                while (iter.hasNext()) {
                    RecordInfo record = iter.next();
                    if (recordsToDelete.contains(record.id)) {
                        iter.remove();
                    }
                }
                recordsToDelete.clear();
                ActiveMQJournalLogger.LOGGER.debug("flush delete done");
            }
        }

        @Override
        public void addPreparedTransaction(final PreparedTransactionInfo preparedTransaction) {
            preparedTransactions.add(preparedTransaction);
            checkDeleteSize();
        }

        @Override
        public void addRecord(final RecordInfo info) {
            records.add(info);
            checkDeleteSize();
        }

        @Override
        public void updateRecord(final RecordInfo info) {
            records.add(info);
            checkDeleteSize();
        }

        @Override
        public void deleteRecord(final long id) {
            recordsToDelete.add(id);
            checkDeleteSize();
        }

        @Override
        public void failedTransaction(final long transactionID, final List<RecordInfo> records, final List<RecordInfo> recordsToDelete) {
            if (failureCallback != null) {
                failureCallback.failedTransaction(transactionID, records, recordsToDelete);
            }
        }
    }, fixBadTX, null);
    for (RecordInfo record : records) {
        if (!recordsToDelete.contains(record.id)) {
            committedRecords.add(record);
        }
    }
    return info;
}

Example 53 with RecordInfo

use of org.apache.activemq.artemis.core.journal.RecordInfo in project activemq-artemis by apache.

the class JournalTptBenchmark method main.

public static void main(String[] args) throws Exception {
    final boolean useDefaultIoExecutor = true;
    final int fileSize = 10 * 1024 * 1024;
    final boolean dataSync = false;
    final Type type = Type.Mapped;
    final int tests = 10;
    final int warmup = 20_000;
    final int measurements = 100_000;
    final int msgSize = 100;
    final byte[] msgContent = new byte[msgSize];
    Arrays.fill(msgContent, (byte) 1);
    final int totalMessages = (measurements * tests + warmup);
    final File tmpDirectory = new File("./");
    // using the default configuration when the broker starts!
    final SequentialFileFactory factory;
    switch(type) {
        case Mapped:
            factory = new MappedSequentialFileFactory(tmpDirectory, fileSize, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, null).setDatasync(dataSync);
            break;
        case Nio:
            factory = new NIOSequentialFileFactory(tmpDirectory, true, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_NIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_NIO, 1, false, null, null).setDatasync(dataSync);
            break;
        case Aio:
            factory = new AIOSequentialFileFactory(tmpDirectory, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_SIZE_AIO, ArtemisConstants.DEFAULT_JOURNAL_BUFFER_TIMEOUT_AIO, 500, false, null, null).setDatasync(dataSync);
            // disable it when using directly the same buffer: ((AIOSequentialFileFactory)factory).disableBufferReuse();
            if (!LibaioContext.isLoaded()) {
                throw new IllegalStateException("lib AIO not loaded!");
            }
            break;
        default:
            throw new AssertionError("unsupported case");
    }
    int numFiles = (int) (totalMessages * factory.calculateBlockSize(msgSize)) / fileSize;
    if (numFiles < 2) {
        numFiles = 2;
    }
    ExecutorService service = null;
    final Journal journal;
    if (useDefaultIoExecutor) {
        journal = new JournalImpl(fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO());
        journal.start();
    } else {
        final ArrayList<MpscArrayQueue<Runnable>> tasks = new ArrayList<>();
        service = Executors.newSingleThreadExecutor();
        journal = new JournalImpl(() -> new ArtemisExecutor() {

            private final MpscArrayQueue<Runnable> taskQueue = new MpscArrayQueue<>(1024);

            {
                tasks.add(taskQueue);
            }

            @Override
            public void execute(Runnable command) {
                while (!taskQueue.offer(command)) {
                    LockSupport.parkNanos(1L);
                }
            }
        }, fileSize, numFiles, numFiles, Integer.MAX_VALUE, 100, factory, "activemq-data", "amq", factory.getMaxIO(), 0);
        journal.start();
        service.execute(() -> {
            final int size = tasks.size();
            final int capacity = 1024;
            while (!Thread.currentThread().isInterrupted()) {
                for (int i = 0; i < size; i++) {
                    final MpscArrayQueue<Runnable> runnables = tasks.get(i);
                    for (int j = 0; j < capacity; j++) {
                        final Runnable task = runnables.poll();
                        if (task == null) {
                            break;
                        }
                        try {
                            task.run();
                        } catch (Throwable t) {
                            System.err.println(t);
                        }
                    }
                }
            }
        });
    }
    try {
        journal.load(new ArrayList<RecordInfo>(), null, null);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    try {
        final EncodingSupport encodingSupport = new EncodingSupport() {

            @Override
            public int getEncodeSize() {
                return msgSize;
            }

            @Override
            public void encode(ActiveMQBuffer buffer) {
                final int writerIndex = buffer.writerIndex();
                buffer.setBytes(writerIndex, msgContent);
                buffer.writerIndex(writerIndex + msgSize);
            }

            @Override
            public void decode(ActiveMQBuffer buffer) {
            }
        };
        long id = 1;
        {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, warmup);
            id += warmup;
            System.out.println("warmup:" + (measurements * 1000_000_000L) / elapsed + " ops/sec");
        }
        for (int t = 0; t < tests; t++) {
            final long elapsed = writeMeasurements(id, journal, encodingSupport, measurements);
            System.out.println((measurements * 1000_000_000L) / elapsed + " ops/sec");
            id += warmup;
        }
    } finally {
        journal.stop();
        if (service != null) {
            service.shutdown();
        }
        final File[] fileToDeletes = tmpDirectory.listFiles();
        System.out.println("Files to deletes" + Arrays.toString(fileToDeletes));
        Stream.of(fileToDeletes).forEach(File::delete);
    }
}

Example 54 with RecordInfo

use of org.apache.activemq.artemis.core.journal.RecordInfo in project activemq-artemis by apache.

the class BridgeTest method loadQueues.

/**
 * It will inspect the journal directly and determine if there are queues on this journal,
 *
 * @param serverToInvestigate
 * @return a Map containing the reference counts per queue
 * @throws Exception
 */
protected Map<Long, AtomicInteger> loadQueues(ActiveMQServer serverToInvestigate) throws Exception {
    SequentialFileFactory messagesFF = new NIOSequentialFileFactory(serverToInvestigate.getConfiguration().getJournalLocation(), 1);
    JournalImpl messagesJournal = new JournalImpl(serverToInvestigate.getConfiguration().getJournalFileSize(), serverToInvestigate.getConfiguration().getJournalMinFiles(), serverToInvestigate.getConfiguration().getJournalPoolFiles(), 0, 0, messagesFF, "activemq-data", "amq", 1);
    List<RecordInfo> records = new LinkedList<>();
    List<PreparedTransactionInfo> preparedTransactions = new LinkedList<>();
    messagesJournal.start();
    messagesJournal.load(records, preparedTransactions, null);
    // These are more immutable integers
    Map<Long, AtomicInteger> messageRefCounts = new HashMap<>();
    for (RecordInfo info : records) {
        Object o = DescribeJournal.newObjectEncoding(info);
        if (info.getUserRecordType() == JournalRecordIds.ADD_REF) {
            DescribeJournal.ReferenceDescribe ref = (DescribeJournal.ReferenceDescribe) o;
            AtomicInteger count = messageRefCounts.get(ref.refEncoding.queueID);
            if (count == null) {
                count = new AtomicInteger(1);
                messageRefCounts.put(ref.refEncoding.queueID, count);
            } else {
                count.incrementAndGet();
            }
        }
    }
    messagesJournal.stop();
    return messageRefCounts;
}

Example 55 with RecordInfo

use of org.apache.activemq.artemis.core.journal.RecordInfo in project activemq-artemis by apache.

the class JournalCleanupCompactStressTest method reloadJournal.

/**
 * @throws Exception
 */
private void reloadJournal() throws Exception {
    assertEquals(0, errors.get());
    ArrayList<RecordInfo> committedRecords = new ArrayList<>();
    ArrayList<PreparedTransactionInfo> preparedTransactions = new ArrayList<>();
    journal.load(committedRecords, preparedTransactions, new TransactionFailureCallback() {

        @Override
        public void failedTransaction(long transactionID, List<RecordInfo> records, List<RecordInfo> recordsToDelete) {
        }
    });
    long appends = 0, updates = 0;
    for (RecordInfo record : committedRecords) {
        if (record.isUpdate) {
            updates++;
        } else {
            appends++;
        }
    }
    assertEquals(numberOfRecords.get() - numberOfDeletes.get(), appends);
}