Examples with SequentialFileFactory org.apache.activemq.artemis.core.io.SequentialFileFactory used on opensource projects

Example 1 with SequentialFileFactory

use of org.apache.activemq.artemis.core.io.SequentialFileFactory in project activemq-artemis by apache.

the class DescribeJournal method describeBindingsJournal.

public static void describeBindingsJournal(final File bindingsDir, PrintStream out, boolean safe) throws Exception {
    SequentialFileFactory bindingsFF = new NIOSequentialFileFactory(bindingsDir, null, 1);
    JournalImpl bindings = new JournalImpl(1024 * 1024, 2, 2, -1, 0, bindingsFF, "activemq-bindings", "bindings", 1);
    describeJournal(bindingsFF, bindings, bindingsDir, out, safe);
}

Example 2 with SequentialFileFactory

use of org.apache.activemq.artemis.core.io.SequentialFileFactory in project activemq-artemis by apache.

the class DescribeJournal method describeMessagesJournal.

public static DescribeJournal describeMessagesJournal(final File messagesDir, PrintStream out, boolean safe) throws Exception {
    SequentialFileFactory messagesFF = new NIOSequentialFileFactory(messagesDir, null, 1);
    // Will use only default values. The load function should adapt to anything different
    ConfigurationImpl defaultValues = new ConfigurationImpl();
    JournalImpl messagesJournal = new JournalImpl(defaultValues.getJournalFileSize(), defaultValues.getJournalMinFiles(), defaultValues.getJournalPoolFiles(), 0, 0, messagesFF, "activemq-data", "amq", 1);
    return describeJournal(messagesFF, messagesJournal, messagesDir, out, safe);
}

Example 3 with SequentialFileFactory

use of org.apache.activemq.artemis.core.io.SequentialFileFactory in project activemq-artemis by apache.

the class SyncCalculation method syncTest.

/**
 * It will perform {@code tries} write tests of {@code blockSize * blocks} bytes and returning the lowest elapsed time to perform a try.
 *
 * <p>
 * Please configure {@code blocks >= -XX:CompileThreshold} (ie by default on most JVMs is 10000) to favour the best JIT/OSR compilation (ie: Just In Time/On Stack Replacement)
 * if the test is running on a temporary file-system (eg: tmpfs on Linux) or without {@code fsync}.
 * <p>
 * NOTE: The write latencies are provided only if {@code verbose && !(journalType == JournalType.ASYNCIO && !syncWrites)} (ie are used effective synchronous writes).
 *
 * @param datafolder  the folder where the journal files will be stored
 * @param blockSize   the size in bytes of each write on the journal
 * @param blocks      the number of {@code blockSize} writes performed on each try
 * @param tries       the number of tests
 * @param verbose     {@code true} to make the output verbose, {@code false} otherwise
 * @param fsync       if {@code true} the test is performing full durable writes, {@code false} otherwise
 * @param syncWrites  if {@code true} each write is performed only if the previous one is completed, {@code false} otherwise (ie each try will wait only the last write)
 * @param fileName    the name of the journal file used for the test
 * @param maxAIO      the max number of in-flight IO requests (if {@code journalType} will support it)
 * @param journalType the {@link JournalType} used for the tests
 * @return the lowest elapsed time (in {@link TimeUnit#MILLISECONDS}) to perform a try
 * @throws Exception
 */
public static long syncTest(File datafolder, int blockSize, int blocks, int tries, boolean verbose, boolean fsync, boolean syncWrites, String fileName, int maxAIO, JournalType journalType) throws Exception {
    SequentialFileFactory factory = newFactory(datafolder, fsync, journalType, blockSize * blocks, maxAIO);
    final boolean asyncWrites = journalType == JournalType.ASYNCIO && !syncWrites;
    // the write latencies could be taken only when writes are effectively synchronous
    final Histogram writeLatencies = (verbose && !asyncWrites) ? new Histogram(MAX_FLUSH_NANOS, 2) : null;
    if (journalType == JournalType.ASYNCIO && syncWrites) {
        System.out.println();
        System.out.println("*******************************************************************************************");
        System.out.println("*** Notice: The recommendation for AsyncIO journal is to not use --sync-writes          ***");
        System.out.println("***         The measures here will be useful to understand your device                  ***");
        System.out.println("***         however the result here won't represent the best configuration option       ***");
        System.out.println("*******************************************************************************************");
        System.out.println();
    }
    if (verbose) {
        System.out.println("Using " + factory.getClass().getName() + " to calculate sync times, alignment=" + factory.getAlignment());
        if (writeLatencies == null) {
            System.out.println("*** Use --sync-writes if you want to see a histogram for each write performed ***");
        }
    }
    SequentialFile file = factory.createSequentialFile(fileName);
    // to be sure that a process/thread crash won't leave the dataFolder with garbage files
    file.getJavaFile().deleteOnExit();
    try {
        final ByteBuffer bufferBlock = allocateAlignedBlock(blockSize, factory);
        // making sure the blockSize matches the device
        blockSize = bufferBlock.remaining();
        file.delete();
        file.open();
        file.fill(blockSize * blocks);
        file.close();
        long[] result = new long[tries];
        final ReusableLatch latch = new ReusableLatch(0);
        IOCallback callback = new IOCallback() {

            @Override
            public void done() {
                latch.countDown();
            }

            @Override
            public void onError(int errorCode, String errorMessage) {
            }
        };
        DecimalFormat dcformat = new DecimalFormat("###.##");
        for (int ntry = 0; ntry < tries; ntry++) {
            if (verbose) {
                System.out.println("**************************************************");
                System.out.println(ntry + " of " + tries + " calculation");
            }
            file.open();
            file.position(0);
            long start = System.currentTimeMillis();
            for (int i = 0; i < blocks; i++) {
                bufferBlock.position(0);
                latch.countUp();
                long startWrite = 0;
                if (writeLatencies != null) {
                    startWrite = System.nanoTime();
                }
                file.writeDirect(bufferBlock, true, callback);
                if (syncWrites) {
                    flushLatch(latch);
                }
                if (writeLatencies != null) {
                    final long elapsedWriteNanos = System.nanoTime() - startWrite;
                    writeLatencies.recordValue(elapsedWriteNanos);
                }
            }
            if (!syncWrites)
                flushLatch(latch);
            long end = System.currentTimeMillis();
            result[ntry] = (end - start);
            if (verbose) {
                double writesPerMillisecond = (double) blocks / (double) result[ntry];
                System.out.println("Time = " + result[ntry] + " milliseconds");
                System.out.println("Writes / millisecond = " + dcformat.format(writesPerMillisecond));
                System.out.println("bufferTimeout = " + toNanos(result[ntry], blocks, verbose));
                System.out.println("**************************************************");
            }
            file.close();
            if (ntry == 0 && writeLatencies != null) {
                // discarding the first one.. some warmup time
                writeLatencies.reset();
            }
        }
        factory.releaseDirectBuffer(bufferBlock);
        if (writeLatencies != null) {
            System.out.println("Write Latencies Percentile Distribution in microseconds");
            // print latencies in us -> (ns * 1000d)
            System.out.println("*****************************************************************");
            writeLatencies.outputPercentileDistribution(System.out, 1000d);
            System.out.println();
            System.out.println("*****************************************************************");
            System.out.println("*** this may be useful to generate charts if you like charts: ***");
            System.out.println("*** http://hdrhistogram.github.io/HdrHistogram/plotFiles.html ***");
            System.out.println("*****************************************************************");
            System.out.println();
            writeLatencies.reset();
        }
        long totalTime = Long.MAX_VALUE;
        for (int i = 0; i < tries; i++) {
            if (result[i] < totalTime) {
                totalTime = result[i];
            }
        }
        return totalTime;
    } finally {
        try {
            file.close();
        } catch (Exception e) {
        }
        try {
            file.delete();
        } catch (Exception e) {
        }
        try {
            factory.stop();
        } catch (Exception e) {
        }
    }
}

Example 4 with SequentialFileFactory

use of org.apache.activemq.artemis.core.io.SequentialFileFactory in project activemq-artemis by apache.

the class SyncCalculation method newFactory.

private static SequentialFileFactory newFactory(File datafolder, boolean datasync, JournalType journalType, int fileSize, int maxAIO) {
    SequentialFileFactory factory;
    if (journalType == JournalType.ASYNCIO && !LibaioContext.isLoaded()) {
        journalType = JournalType.NIO;
    }
    switch(journalType) {
        case NIO:
            factory = new NIOSequentialFileFactory(datafolder, 1).setDatasync(datasync);
            ((NIOSequentialFileFactory) factory).disableBufferReuse();
            factory.start();
            return factory;
        case ASYNCIO:
            factory = new AIOSequentialFileFactory(datafolder, maxAIO).setDatasync(datasync);
            factory.start();
            ((AIOSequentialFileFactory) factory).disableBufferReuse();
            return factory;
        case MAPPED:
            factory = new MappedSequentialFileFactory(datafolder, fileSize, false, 0, 0, null).setDatasync(datasync).disableBufferReuse();
            factory.start();
            return factory;
        default:
            throw ActiveMQMessageBundle.BUNDLE.invalidJournalType2(journalType);
    }
}

Example 5 with SequentialFileFactory

use of org.apache.activemq.artemis.core.io.SequentialFileFactory in project activemq-artemis by apache.

the class FileIOUtilTest method testCopy.

@Test
public void testCopy() throws Exception {
    System.out.println("Data at " + temporaryFolder.getRoot());
    SequentialFileFactory factory = new NIOSequentialFileFactory(temporaryFolder.getRoot(), 100);
    SequentialFile file = factory.createSequentialFile("file1.bin");
    file.open();
    ByteBuffer buffer = ByteBuffer.allocate(204800);
    buffer.put(new byte[204800]);
    buffer.rewind();
    file.writeDirect(buffer, true);
    buffer = ByteBuffer.allocate(409605);
    buffer.put(new byte[409605]);
    buffer.rewind();
    SequentialFile file2 = factory.createSequentialFile("file2.bin");
    file2.open();
    file2.writeDirect(buffer, true);
    // This is allocating a reusable buffer to perform the copy, just like it's used within LargeMessageInSync
    buffer = ByteBuffer.allocate(4 * 1024);
    SequentialFile newFile = factory.createSequentialFile("file1.cop");
    FileIOUtil.copyData(file, newFile, buffer);
    SequentialFile newFile2 = factory.createSequentialFile("file2.cop");
    FileIOUtil.copyData(file2, newFile2, buffer);
    Assert.assertEquals(file.size(), newFile.size());
    Assert.assertEquals(file2.size(), newFile2.size());
    newFile.close();
    newFile2.close();
    file.close();
    file2.close();
    System.out.println("Test result::");
}