Examples with SequentialFile org.apache.activemq.artemis.core.io.SequentialFile used on opensource projects

Example 21 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class NIOJournalCompactTest method internalCompactTest.

private void internalCompactTest(final boolean preXA, // prepare before compact
final boolean postXA, // prepare after compact
final boolean regularAdd, final boolean performAppend, final boolean performUpdate, boolean performDelete, boolean performNonTransactionalDelete, final boolean pendingTransactions, final boolean deleteTransactRecords, final boolean delayCommit, final boolean createControlFile, final boolean deleteControlFile, final boolean renameFilesAfterCompacting) throws Exception {
    if (performNonTransactionalDelete) {
        performDelete = false;
    }
    if (performDelete) {
        performNonTransactionalDelete = false;
    }
    setup(2, 60 * 4096, false);
    ArrayList<Long> liveIDs = new ArrayList<>();
    ArrayList<Pair<Long, Long>> transactedRecords = new ArrayList<>();
    final CountDownLatch latchDone = new CountDownLatch(1);
    final CountDownLatch latchWait = new CountDownLatch(1);
    journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, fileFactory, filePrefix, fileExtension, maxAIO) {

        @Override
        protected SequentialFile createControlFile(final List<JournalFile> files, final List<JournalFile> newFiles, final Pair<String, String> pair) throws Exception {
            if (createControlFile) {
                return super.createControlFile(files, newFiles, pair);
            } else {
                throw new IllegalStateException("Simulating a crash during compact creation");
            }
        }

        @Override
        protected void deleteControlFile(final SequentialFile controlFile) throws Exception {
            if (deleteControlFile) {
                super.deleteControlFile(controlFile);
            }
        }

        @Override
        protected void renameFiles(final List<JournalFile> oldFiles, final List<JournalFile> newFiles) throws Exception {
            if (renameFilesAfterCompacting) {
                super.renameFiles(oldFiles, newFiles);
            }
        }

        @Override
        public void onCompactDone() {
            latchDone.countDown();
            System.out.println("Waiting on Compact");
            try {
                ActiveMQTestBase.waitForLatch(latchWait);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            System.out.println("Done");
        }
    };
    journal.setAutoReclaim(false);
    startJournal();
    load();
    long transactionID = 0;
    if (regularAdd) {
        for (int i = 0; i < NIOJournalCompactTest.NUMBER_OF_RECORDS / 2; i++) {
            add(i);
            if (i % 10 == 0 && i > 0) {
                journal.forceMoveNextFile();
            }
            update(i);
        }
        for (int i = NIOJournalCompactTest.NUMBER_OF_RECORDS / 2; i < NIOJournalCompactTest.NUMBER_OF_RECORDS; i++) {
            addTx(transactionID, i);
            updateTx(transactionID, i);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
            commit(transactionID++);
            update(i);
        }
    }
    if (pendingTransactions) {
        for (long i = 0; i < 100; i++) {
            long recordID = idGenerator.generateID();
            addTx(transactionID, recordID);
            updateTx(transactionID, recordID);
            if (preXA) {
                prepare(transactionID, new SimpleEncoding(10, (byte) 0));
            }
            transactedRecords.add(new Pair<>(transactionID++, recordID));
        }
    }
    if (regularAdd) {
        for (int i = 0; i < NIOJournalCompactTest.NUMBER_OF_RECORDS; i++) {
            if (!(i % 10 == 0)) {
                delete(i);
            } else {
                liveIDs.add((long) i);
            }
        }
    }
    journal.forceMoveNextFile();
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                journal.testCompact();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    };
    t.start();
    ActiveMQTestBase.waitForLatch(latchDone);
    int nextID = NIOJournalCompactTest.NUMBER_OF_RECORDS;
    if (performAppend) {
        for (int i = 0; i < 50; i++) {
            add(nextID++);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
        }
        for (int i = 0; i < 50; i++) {
            // A Total new transaction (that was created after the compact started) to add new record while compacting
            // is still working
            addTx(transactionID, nextID++);
            commit(transactionID++);
            if (i % 10 == 0) {
                journal.forceMoveNextFile();
            }
        }
    }
    if (performUpdate) {
        int count = 0;
        for (Long liveID : liveIDs) {
            if (count++ % 2 == 0) {
                update(liveID);
            } else {
                // A Total new transaction (that was created after the compact started) to update a record that is being
                // compacted
                updateTx(transactionID, liveID);
                commit(transactionID++);
            }
        }
    }
    if (performDelete) {
        int count = 0;
        for (long liveID : liveIDs) {
            if (count++ % 2 == 0) {
                System.out.println("Deleting no trans " + liveID);
                delete(liveID);
            } else {
                System.out.println("Deleting TX " + liveID);
                // A Total new transaction (that was created after the compact started) to delete a record that is being
                // compacted
                deleteTx(transactionID, liveID);
                commit(transactionID++);
            }
            System.out.println("Deletes are going into " + ((JournalImpl) journal).getCurrentFile());
        }
    }
    if (performNonTransactionalDelete) {
        for (long liveID : liveIDs) {
            delete(liveID);
        }
    }
    if (pendingTransactions && !delayCommit) {
        for (Pair<Long, Long> tx : transactedRecords) {
            if (postXA) {
                prepare(tx.getA(), new SimpleEncoding(10, (byte) 0));
            }
            if (tx.getA() % 2 == 0) {
                commit(tx.getA());
                if (deleteTransactRecords) {
                    delete(tx.getB());
                }
            } else {
                rollback(tx.getA());
            }
        }
    }
    /**
     * Some independent adds and updates
     */
    for (int i = 0; i < 1000; i++) {
        long id = idGenerator.generateID();
        add(id);
        delete(id);
        if (i % 100 == 0) {
            journal.forceMoveNextFile();
        }
    }
    journal.forceMoveNextFile();
    latchWait.countDown();
    t.join();
    if (pendingTransactions && delayCommit) {
        for (Pair<Long, Long> tx : transactedRecords) {
            if (postXA) {
                prepare(tx.getA(), new SimpleEncoding(10, (byte) 0));
            }
            if (tx.getA() % 2 == 0) {
                commit(tx.getA());
                if (deleteTransactRecords) {
                    delete(tx.getB());
                }
            } else {
                rollback(tx.getA());
            }
        }
    }
    long lastId = idGenerator.generateID();
    add(lastId);
    if (createControlFile && deleteControlFile && renameFilesAfterCompacting) {
        journal.testCompact();
    }
    journal.flush();
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
    journal.forceMoveNextFile();
    update(lastId);
    stopJournal();
    createJournal();
    startJournal();
    loadAndCheck();
}

Example 22 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class AIOSequentialFileFactoryTest method testBuffer.

@Test
public void testBuffer() throws Exception {
    SequentialFile file = factory.createSequentialFile("filtetmp.log");
    file.open();
    ByteBuffer buff = factory.newBuffer(10);
    Assert.assertEquals(factory.getAlignment(), buff.limit());
    file.close();
    factory.releaseBuffer(buff);
}

Example 23 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class JournalStorageManager method cleanupIncompleteFiles.

private void cleanupIncompleteFiles() throws Exception {
    if (largeMessagesFactory != null) {
        List<String> tmpFiles = largeMessagesFactory.listFiles("tmp");
        for (String tmpFile : tmpFiles) {
            SequentialFile file = largeMessagesFactory.createSequentialFile(tmpFile);
            file.delete();
        }
    }
}

Example 24 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class JournalStorageManager method parseLargeMessage.

@Override
protected /**
 * @param messages
 * @param buff
 * @return
 * @throws Exception
 */
LargeServerMessage parseLargeMessage(final Map<Long, Message> messages, final ActiveMQBuffer buff) throws Exception {
    LargeServerMessage largeMessage = createLargeMessage();
    LargeMessagePersister.getInstance().decode(buff, largeMessage);
    if (largeMessage.containsProperty(Message.HDR_ORIG_MESSAGE_ID)) {
        // for compatibility: couple with old behaviour, copying the old file to avoid message loss
        long originalMessageID = largeMessage.getLongProperty(Message.HDR_ORIG_MESSAGE_ID);
        SequentialFile currentFile = createFileForLargeMessage(largeMessage.getMessageID(), true);
        if (!currentFile.exists()) {
            SequentialFile linkedFile = createFileForLargeMessage(originalMessageID, true);
            if (linkedFile.exists()) {
                linkedFile.copyTo(currentFile);
                linkedFile.close();
            }
        }
        currentFile.close();
    }
    return largeMessage;
}

Example 25 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class JournalStorageManager method recoverPendingLargeMessages.

/**
 * Sets a list of large message files into the replicationManager for synchronization.
 * <p>
 * Collects a list of existing large messages and their current size, passing re.
 * <p>
 * So we know how much of a given message to sync with the backup. Further data appends to the
 * messages will be replicated normally.
 *
 * @throws Exception
 */
private Map<Long, Pair<String, Long>> recoverPendingLargeMessages() throws Exception {
    Map<Long, Pair<String, Long>> largeMessages = new HashMap<>();
    // only send durable messages... // listFiles append a "." to anything...
    List<String> filenames = largeMessagesFactory.listFiles("msg");
    List<Long> idList = new ArrayList<>();
    for (String filename : filenames) {
        Long id = getLargeMessageIdFromFilename(filename);
        if (!largeMessagesToDelete.contains(id)) {
            idList.add(id);
            SequentialFile seqFile = largeMessagesFactory.createSequentialFile(filename);
            long size = seqFile.size();
            largeMessages.put(id, new Pair<>(filename, size));
        }
    }
    return largeMessages;
}