Examples with JournalFile org.apache.activemq.artemis.core.journal.impl.JournalFile used on opensource projects

Example 1 with JournalFile

use of org.apache.activemq.artemis.core.journal.impl.JournalFile in project activemq-artemis by apache.

the class ReplicationEndpoint method handleStartReplicationSynchronization.

/**
 * Reserves files (with the given fileID) in the specified journal, and places a
 * {@link FileWrapperJournal} in place to store messages while synchronization is going on.
 *
 * @param packet
 * @return if the incoming packet indicates the synchronization is finished then return an acknowledgement otherwise
 * return an empty response
 * @throws Exception
 */
private ReplicationResponseMessageV2 handleStartReplicationSynchronization(final ReplicationStartSyncMessage packet) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("handleStartReplicationSynchronization:: nodeID = " + packet);
    }
    ReplicationResponseMessageV2 replicationResponseMessage = new ReplicationResponseMessageV2();
    if (!started)
        return replicationResponseMessage;
    if (packet.isSynchronizationFinished()) {
        finishSynchronization(packet.getNodeID());
        replicationResponseMessage.setSynchronizationIsFinishedAcknowledgement(true);
        return replicationResponseMessage;
    }
    switch(packet.getDataType()) {
        case LargeMessages:
            for (long msgID : packet.getFileIds()) {
                createLargeMessage(msgID, true);
            }
            break;
        case JournalBindings:
        case JournalMessages:
            if (wantedFailBack && !packet.isServerToFailBack()) {
                ActiveMQServerLogger.LOGGER.autoFailBackDenied();
            }
            final JournalContent journalContent = SyncDataType.getJournalContentType(packet.getDataType());
            final Journal journal = journalsHolder.get(journalContent);
            if (packet.getNodeID() != null) {
                // At the start of replication, we still do not know which is the nodeID that the live uses.
                // This is the point where the backup gets this information.
                backupQuorum.liveIDSet(packet.getNodeID());
            }
            Map<Long, JournalSyncFile> mapToFill = filesReservedForSync.get(journalContent);
            for (Entry<Long, JournalFile> entry : journal.createFilesForBackupSync(packet.getFileIds()).entrySet()) {
                mapToFill.put(entry.getKey(), new JournalSyncFile(entry.getValue()));
            }
            FileWrapperJournal syncJournal = new FileWrapperJournal(journal);
            registerJournal(journalContent.typeByte, syncJournal);
            break;
        default:
            throw ActiveMQMessageBundle.BUNDLE.replicationUnhandledDataType();
    }
    return replicationResponseMessage;
}

Example 2 with JournalFile

use of org.apache.activemq.artemis.core.journal.impl.JournalFile in project activemq-artemis by apache.

the class ReclaimerTest method setupPosNeg.

private void setupPosNeg(final int fileNumber, final int pos, final int... neg) {
    JournalFile file = files[fileNumber];
    int totalDep = file.getTotalNegativeToOthers();
    for (int i = 0; i < pos; i++) {
        file.incPosCount();
    }
    for (int i = 0; i < neg.length; i++) {
        JournalFile reclaimable2 = files[i];
        for (int j = 0; j < neg[i]; j++) {
            file.incNegCount(reclaimable2);
            totalDep++;
        }
    }
    assertEquals(totalDep, file.getTotalNegativeToOthers());
}

Example 3 with JournalFile

use of org.apache.activemq.artemis.core.journal.impl.JournalFile in project activemq-artemis by apache.

the class NIOJournalCompactTest method internalCompactTest.

private void internalCompactTest(final boolean preXA, // prepare before compact
final boolean postXA, // prepare after compact
final boolean regularAdd, final boolean performAppend, final boolean performUpdate, boolean performDelete, boolean performNonTransactionalDelete, final boolean pendingTransactions, final boolean deleteTransactRecords, final boolean delayCommit, final boolean createControlFile, final boolean deleteControlFile, final boolean renameFilesAfterCompacting) throws Exception {
    if (performNonTransactionalDelete) {
        performDelete = false;
    }
    if (performDelete) {
        performNonTransactionalDelete = false;
    }
    setup(2, 60 * 4096, false);
    ArrayList<Long> liveIDs = new ArrayList<>();
    ArrayList<Pair<Long, Long>> transactedRecords = new ArrayList<>();
    final CountDownLatch latchDone = new CountDownLatch(1);
    final CountDownLatch latchWait = new CountDownLatch(1);
    journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, fileFactory, filePrefix, fileExtension, maxAIO) {

        @Override
        protected SequentialFile createControlFile(final List<JournalFile> files, final List<JournalFile> newFiles, final Pair<String, String> pair) throws Exception {
            if (createControlFile) {
                return super.createControlFile(files, newFiles, pair);
            } else {
                throw new IllegalStateException("Simulating a crash during compact creation");
            }
        }

        @Override
        protected void deleteControlFile(final SequentialFile controlFile) throws Exception {
            if (deleteControlFile) {
                super.deleteControlFile(controlFile);
            }
        }

        @Override
        protected void renameFiles(final List<JournalFile> oldFiles, final List<JournalFile> newFiles) throws Exception {
            if (renameFilesAfterCompacting) {
                super.renameFiles(oldFiles, newFiles);
            }
        }

        @Override
        public void onCompactDone() {
            latchDone.countDown();
            System.out.println("Waiting on Compact");
            try {
                ActiveMQTestBase.waitForLatch(latchWait);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            System.out.println("Done");
        }
    };
    journal.setAutoReclaim(false);
    startJournal();
    load();
    long transactionID = 0;
    if (regularAdd) {
        for (int i = 0; i < NIOJournalCompactTest.NUMBER_OF_RECORDS / 2; i++) {
            add(i);
            if (i % 10 == 0 && i > 0) {
                journal.forceMoveNextFile();
            }
            update(i);
        }
        for (int i = NIOJournalCompactTest.NUMBER_OF_RECORDS / 2; i < NIOJournalCompactTest.NUMBER_OF_RECORDS; i++) {
            addTx(transactionID, i);
            updateTx(transactionID, i);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
            commit(transactionID++);
            update(i);
        }
    }
    if (pendingTransactions) {
        for (long i = 0; i < 100; i++) {
            long recordID = idGenerator.generateID();
            addTx(transactionID, recordID);
            updateTx(transactionID, recordID);
            if (preXA) {
                prepare(transactionID, new SimpleEncoding(10, (byte) 0));
            }
            transactedRecords.add(new Pair<>(transactionID++, recordID));
        }
    }
    if (regularAdd) {
        for (int i = 0; i < NIOJournalCompactTest.NUMBER_OF_RECORDS; i++) {
            if (!(i % 10 == 0)) {
                delete(i);
            } else {
                liveIDs.add((long) i);
            }
        }
    }
    journal.forceMoveNextFile();
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                journal.testCompact();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    t.start();
    ActiveMQTestBase.waitForLatch(latchDone);
    int nextID = NIOJournalCompactTest.NUMBER_OF_RECORDS;
    if (performAppend) {
        for (int i = 0; i < 50; i++) {
            add(nextID++);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
        }
        for (int i = 0; i < 50; i++) {
            // A Total new transaction (that was created after the compact started) to add new record while compacting
            // is still working
            addTx(transactionID, nextID++);
            commit(transactionID++);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
        }
    }
    if (performUpdate) {
        int count = 0;
        for (Long liveID : liveIDs) {
            if (count++ % 2 == 0) {
                update(liveID);
            } else {
                // A Total new transaction (that was created after the compact started) to update a record that is being
                // compacted
                updateTx(transactionID, liveID);
                commit(transactionID++);
            }
        }
    }
    if (performDelete) {
        int count = 0;
        for (long liveID : liveIDs) {
            if (count++ % 2 == 0) {
                System.out.println("Deleting no trans " + liveID);
                delete(liveID);
            } else {
                System.out.println("Deleting TX " + liveID);
                // A Total new transaction (that was created after the compact started) to delete a record that is being
                // compacted
                deleteTx(transactionID, liveID);
                commit(transactionID++);
            }
            System.out.println("Deletes are going into " + ((JournalImpl) journal).getCurrentFile());
        }
    }
    if (performNonTransactionalDelete) {
        for (long liveID : liveIDs) {
            delete(liveID);
        }
    }
    if (pendingTransactions && !delayCommit) {
        for (Pair<Long, Long> tx : transactedRecords) {
            if (postXA) {
                prepare(tx.getA(), new SimpleEncoding(10, (byte) 0));
            }
            if (tx.getA() % 2 == 0) {
                commit(tx.getA());
                if (deleteTransactRecords) {
                    delete(tx.getB());
                }
            } else {
                rollback(tx.getA());
            }
        }
    }
    /**
     * Some independent adds and updates
     */
    for (int i = 0; i < 1000; i++) {
        long id = idGenerator.generateID();
        add(id);
        delete(id);
        if (i % 100 == 0) {
            journal.forceMoveNextFile();
        }
    }
    journal.forceMoveNextFile();
    latchWait.countDown();
    t.join();
    if (pendingTransactions && delayCommit) {
        for (Pair<Long, Long> tx : transactedRecords) {
            if (postXA) {
                prepare(tx.getA(), new SimpleEncoding(10, (byte) 0));
            }
            if (tx.getA() % 2 == 0) {
                commit(tx.getA());
                if (deleteTransactRecords) {
                    delete(tx.getB());
                }
            } else {
                rollback(tx.getA());
            }
        }
    }
    long lastId = idGenerator.generateID();
    add(lastId);
    if (createControlFile && deleteControlFile && renameFilesAfterCompacting) {
        journal.testCompact();
    }
    journal.flush();
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
    journal.forceMoveNextFile();
    update(lastId);
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
}

Example 4 with JournalFile

use of org.apache.activemq.artemis.core.journal.impl.JournalFile in project activemq-artemis by apache.

the class NIOJournalCompactTest method testLiveSize.

@Test
public void testLiveSize() throws Exception {
    setup(2, 60 * 1024, true);
    createJournal();
    startJournal();
    loadAndCheck();
    ArrayList<Long> listToDelete = new ArrayList<>();
    ArrayList<Integer> expectedSizes = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        long id = idGenerator.generateID();
        listToDelete.add(id);
        expectedSizes.add(recordLength + JournalImpl.SIZE_ADD_RECORD + 1);
        add(id);
        journal.forceMoveNextFile();
        update(id);
        expectedSizes.add(recordLength + JournalImpl.SIZE_ADD_RECORD + 1);
        journal.forceMoveNextFile();
    }
    JournalFile[] files = journal.getDataFiles();
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
    journal.forceMoveNextFile();
    JournalFile[] files2 = journal.getDataFiles();
    Assert.assertEquals(files.length, files2.length);
    for (int i = 0; i < files.length; i++) {
        Assert.assertEquals(expectedSizes.get(i).intValue(), files[i].getLiveSize());
        Assert.assertEquals(expectedSizes.get(i).intValue(), files2[i].getLiveSize());
    }
    for (long id : listToDelete) {
        delete(id);
    }
    journal.forceMoveNextFile();
    JournalFile[] files3 = journal.getDataFiles();
    for (JournalFile file : files3) {
        Assert.assertEquals(0, file.getLiveSize());
    }
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
    files3 = journal.getDataFiles();
    for (JournalFile file : files3) {
        Assert.assertEquals(0, file.getLiveSize());
    }
}

Example 5 with JournalFile

use of org.apache.activemq.artemis.core.journal.impl.JournalFile in project activemq-artemis by apache.

the class ActiveMQTestBase method countBindingJournal.

protected HashMap<Integer, AtomicInteger> countBindingJournal(Configuration config) throws Exception {
    final HashMap<Integer, AtomicInteger> recordsType = new HashMap<>();
    SequentialFileFactory messagesFF = new NIOSequentialFileFactory(config.getBindingsLocation(), null, 1);
    JournalImpl messagesJournal = new JournalImpl(config.getJournalFileSize(), config.getJournalMinFiles(), config.getJournalPoolFiles(), 0, 0, messagesFF, "activemq-bindings", "bindings", 1);
    List<JournalFile> filesToRead = messagesJournal.orderFiles();
    for (JournalFile file : filesToRead) {
        JournalImpl.readJournalFile(messagesFF, file, new RecordTypeCounter(recordsType));
    }
    return recordsType;
}