Examples with SequentialFile org.apache.activemq.artemis.core.io.SequentialFile used on opensource projects

Example 6 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class ReplicationManager method syncJournalFile.

/**
 * Sends the whole content of the file to be duplicated.
 *
 * @throws ActiveMQException
 * @throws Exception
 */
public void syncJournalFile(JournalFile jf, AbstractJournalStorageManager.JournalContent content) throws Exception {
    if (!enabled) {
        return;
    }
    SequentialFile file = jf.getFile().cloneFile();
    try {
        ActiveMQServerLogger.LOGGER.replicaSyncFile(file, file.size());
        sendLargeFile(content, null, jf.getFileID(), file, Long.MAX_VALUE);
    } finally {
        if (file.isOpen())
            file.close();
    }
}

Example 7 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class SyncCalculation method syncTest.

/**
 * It will perform {@code tries} write tests of {@code blockSize * blocks} bytes and returning the lowest elapsed time to perform a try.
 *
 * <p>
 * Please configure {@code blocks >= -XX:CompileThreshold} (ie by default on most JVMs is 10000) to favour the best JIT/OSR compilation (ie: Just In Time/On Stack Replacement)
 * if the test is running on a temporary file-system (eg: tmpfs on Linux) or without {@code fsync}.
 * <p>
 * NOTE: The write latencies are provided only if {@code verbose && !(journalType == JournalType.ASYNCIO && !syncWrites)} (ie are used effective synchronous writes).
 *
 * @param datafolder  the folder where the journal files will be stored
 * @param blockSize   the size in bytes of each write on the journal
 * @param blocks      the number of {@code blockSize} writes performed on each try
 * @param tries       the number of tests
 * @param verbose     {@code true} to make the output verbose, {@code false} otherwise
 * @param fsync       if {@code true} the test is performing full durable writes, {@code false} otherwise
 * @param syncWrites  if {@code true} each write is performed only if the previous one is completed, {@code false} otherwise (ie each try will wait only the last write)
 * @param fileName    the name of the journal file used for the test
 * @param maxAIO      the max number of in-flight IO requests (if {@code journalType} will support it)
 * @param journalType the {@link JournalType} used for the tests
 * @return the lowest elapsed time (in {@link TimeUnit#MILLISECONDS}) to perform a try
 * @throws Exception
 */
public static long syncTest(File datafolder, int blockSize, int blocks, int tries, boolean verbose, boolean fsync, boolean syncWrites, String fileName, int maxAIO, JournalType journalType) throws Exception {
    SequentialFileFactory factory = newFactory(datafolder, fsync, journalType, blockSize * blocks, maxAIO);
    final boolean asyncWrites = journalType == JournalType.ASYNCIO && !syncWrites;
    // the write latencies could be taken only when writes are effectively synchronous
    final Histogram writeLatencies = (verbose && !asyncWrites) ? new Histogram(MAX_FLUSH_NANOS, 2) : null;
    if (journalType == JournalType.ASYNCIO && syncWrites) {
        System.out.println();
        System.out.println("*******************************************************************************************");
        System.out.println("*** Notice: The recommendation for AsyncIO journal is to not use --sync-writes          ***");
        System.out.println("***         The measures here will be useful to understand your device                  ***");
        System.out.println("***         however the result here won't represent the best configuration option       ***");
        System.out.println("*******************************************************************************************");
        System.out.println();
    }
    if (verbose) {
        System.out.println("Using " + factory.getClass().getName() + " to calculate sync times, alignment=" + factory.getAlignment());
        if (writeLatencies == null) {
            System.out.println("*** Use --sync-writes if you want to see a histogram for each write performed ***");
        }
    }
    SequentialFile file = factory.createSequentialFile(fileName);
    // to be sure that a process/thread crash won't leave the dataFolder with garbage files
    file.getJavaFile().deleteOnExit();
    try {
        final ByteBuffer bufferBlock = allocateAlignedBlock(blockSize, factory);
        // making sure the blockSize matches the device
        blockSize = bufferBlock.remaining();
        file.delete();
        file.open();
        file.fill(blockSize * blocks);
        file.close();
        long[] result = new long[tries];
        final ReusableLatch latch = new ReusableLatch(0);
        IOCallback callback = new IOCallback() {

            @Override
            public void done() {
                latch.countDown();
            }

            @Override
            public void onError(int errorCode, String errorMessage) {
            }
        };
        DecimalFormat dcformat = new DecimalFormat("###.##");
        for (int ntry = 0; ntry < tries; ntry++) {
            if (verbose) {
                System.out.println("**************************************************");
                System.out.println(ntry + " of " + tries + " calculation");
            }
            file.open();
            file.position(0);
            long start = System.currentTimeMillis();
            for (int i = 0; i < blocks; i++) {
                bufferBlock.position(0);
                latch.countUp();
                long startWrite = 0;
                if (writeLatencies != null) {
                    startWrite = System.nanoTime();
                }
                file.writeDirect(bufferBlock, true, callback);
                if (syncWrites) {
                    flushLatch(latch);
                }
                if (writeLatencies != null) {
                    final long elapsedWriteNanos = System.nanoTime() - startWrite;
                    writeLatencies.recordValue(elapsedWriteNanos);
                }
            }
            if (!syncWrites)
                flushLatch(latch);
            long end = System.currentTimeMillis();
            result[ntry] = (end - start);
            if (verbose) {
                double writesPerMillisecond = (double) blocks / (double) result[ntry];
                System.out.println("Time = " + result[ntry] + " milliseconds");
                System.out.println("Writes / millisecond = " + dcformat.format(writesPerMillisecond));
                System.out.println("bufferTimeout = " + toNanos(result[ntry], blocks, verbose));
                System.out.println("**************************************************");
            }
            file.close();
            if (ntry == 0 && writeLatencies != null) {
                // discarding the first one.. some warmup time
                writeLatencies.reset();
            }
        }
        factory.releaseDirectBuffer(bufferBlock);
        if (writeLatencies != null) {
            System.out.println("Write Latencies Percentile Distribution in microseconds");
            // print latencies in us -> (ns * 1000d)
            System.out.println("*****************************************************************");
            writeLatencies.outputPercentileDistribution(System.out, 1000d);
            System.out.println();
            System.out.println("*****************************************************************");
            System.out.println("*** this may be useful to generate charts if you like charts: ***");
            System.out.println("*** http://hdrhistogram.github.io/HdrHistogram/plotFiles.html ***");
            System.out.println("*****************************************************************");
            System.out.println();
            writeLatencies.reset();
        }
        long totalTime = Long.MAX_VALUE;
        for (int i = 0; i < tries; i++) {
            if (result[i] < totalTime) {
                totalTime = result[i];
            }
        }
        return totalTime;
    } finally {
        try {
            file.close();
        } catch (Exception e) {
        }
        try {
            file.delete();
        } catch (Exception e) {
        }
        try {
            factory.stop();
        } catch (Exception e) {
        }
    }
}

Example 8 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class JournalFilesRepository method takeFile.

/**
 * This will get a File from freeFile without initializing it
 *
 * @return uninitialized JournalFile
 * @throws Exception
 * @see JournalImpl#initFileHeader(SequentialFileFactory, SequentialFile, int, long)
 */
public JournalFile takeFile(final boolean keepOpened, final boolean multiAIO, final boolean initFile, final boolean tmpCompactExtension) throws Exception {
    JournalFile nextFile = null;
    nextFile = freeFiles.poll();
    if (nextFile != null) {
        freeFilesCount.getAndDecrement();
    }
    if (nextFile == null) {
        nextFile = createFile(keepOpened, multiAIO, initFile, tmpCompactExtension, -1);
    } else {
        if (tmpCompactExtension) {
            SequentialFile sequentialFile = nextFile.getFile();
            sequentialFile.renameTo(sequentialFile.getFileName() + ".cmp");
        }
        if (keepOpened) {
            openFile(nextFile, multiAIO);
        }
    }
    return nextFile;
}

Example 9 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class FileIOUtilTest method testCopy.

@Test
public void testCopy() throws Exception {
    System.out.println("Data at " + temporaryFolder.getRoot());
    SequentialFileFactory factory = new NIOSequentialFileFactory(temporaryFolder.getRoot(), 100);
    SequentialFile file = factory.createSequentialFile("file1.bin");
    file.open();
    ByteBuffer buffer = ByteBuffer.allocate(204800);
    buffer.put(new byte[204800]);
    buffer.rewind();
    file.writeDirect(buffer, true);
    buffer = ByteBuffer.allocate(409605);
    buffer.put(new byte[409605]);
    buffer.rewind();
    SequentialFile file2 = factory.createSequentialFile("file2.bin");
    file2.open();
    file2.writeDirect(buffer, true);
    // This is allocating a reusable buffer to perform the copy, just like it's used within LargeMessageInSync
    buffer = ByteBuffer.allocate(4 * 1024);
    SequentialFile newFile = factory.createSequentialFile("file1.cop");
    FileIOUtil.copyData(file, newFile, buffer);
    SequentialFile newFile2 = factory.createSequentialFile("file2.cop");
    FileIOUtil.copyData(file2, newFile2, buffer);
    Assert.assertEquals(file.size(), newFile.size());
    Assert.assertEquals(file2.size(), newFile2.size());
    newFile.close();
    newFile2.close();
    file.close();
    file2.close();
    System.out.println("Test result::");
}

Example 10 with SequentialFile

use of org.apache.activemq.artemis.core.io.SequentialFile in project activemq-artemis by apache.

the class JournalImpl method checkControlFile.

/**
 * @throws Exception
 */
private void checkControlFile() throws Exception {
    ArrayList<String> dataFiles = new ArrayList<>();
    ArrayList<String> newFiles = new ArrayList<>();
    ArrayList<Pair<String, String>> renames = new ArrayList<>();
    SequentialFile controlFile = AbstractJournalUpdateTask.readControlFile(fileFactory, dataFiles, newFiles, renames);
    if (controlFile != null) {
        for (String dataFile : dataFiles) {
            SequentialFile file = fileFactory.createSequentialFile(dataFile);
            if (file.exists()) {
                file.delete();
            }
        }
        for (String newFile : newFiles) {
            SequentialFile file = fileFactory.createSequentialFile(newFile);
            if (file.exists()) {
                final String originalName = file.getFileName();
                final String newName = originalName.substring(0, originalName.lastIndexOf(".cmp"));
                file.renameTo(newName);
            }
        }
        for (Pair<String, String> rename : renames) {
            SequentialFile fileTmp = fileFactory.createSequentialFile(rename.getA());
            SequentialFile fileTo = fileFactory.createSequentialFile(rename.getB());
            // delete a valid file depending on where the crash occurred during the control file delete
            if (fileTmp.exists()) {
                fileTo.delete();
                fileTmp.renameTo(rename.getB());
            }
        }
        controlFile.delete();
    }
    cleanupTmpFiles(".cmp");
    cleanupTmpFiles(".tmp");
    return;
}