Search in sources :

Example 1 with FileChannel

use of java.nio.channels.FileChannel in project camel by apache.

the class FileLockExclusiveReadLockStrategy method acquireExclusiveReadLock.

@Override
public boolean acquireExclusiveReadLock(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception {
    // must call super
    if (!super.acquireExclusiveReadLock(operations, file, exchange)) {
        return false;
    }
    File target = new File(file.getAbsoluteFilePath());
    LOG.trace("Waiting for exclusive read lock to file: {}", target);
    FileChannel channel = null;
    RandomAccessFile randomAccessFile = null;
    boolean exclusive = false;
    FileLock lock = null;
    try {
        randomAccessFile = new RandomAccessFile(target, "rw");
        // try to acquire rw lock on the file before we can consume it
        channel = randomAccessFile.getChannel();
        StopWatch watch = new StopWatch();
        while (!exclusive) {
            // timeout check
            if (timeout > 0) {
                long delta = watch.taken();
                if (delta > timeout) {
                    CamelLogger.log(LOG, readLockLoggingLevel, "Cannot acquire read lock within " + timeout + " millis. Will skip the file: " + target);
                    // we could not get the lock within the timeout period, so return false
                    return false;
                }
            }
            // get the lock using either try lock or not depending on if we are using timeout or not
            try {
                lock = timeout > 0 ? channel.tryLock() : channel.lock();
            } catch (IllegalStateException ex) {
            // Also catch the OverlappingFileLockException here. Do nothing here                    
            }
            if (lock != null) {
                LOG.trace("Acquired exclusive read lock: {} to file: {}", lock, target);
                exclusive = true;
            } else {
                boolean interrupted = sleep();
                if (interrupted) {
                    // we were interrupted while sleeping, we are likely being shutdown so return false
                    return false;
                }
            }
        }
    } catch (IOException e) {
        // such as AntiVirus or MS Office that has special locks for it's supported files
        if (timeout == 0) {
            // if not using timeout, then we cant retry, so return false
            return false;
        }
        LOG.debug("Cannot acquire read lock. Will try again.", e);
        boolean interrupted = sleep();
        if (interrupted) {
            // we were interrupted while sleeping, we are likely being shutdown so return false
            return false;
        }
    } finally {
        // close channels if we did not grab the lock
        if (!exclusive) {
            IOHelper.close(channel, "while acquiring exclusive read lock for file: " + target, LOG);
            IOHelper.close(randomAccessFile, "while acquiring exclusive read lock for file: " + target, LOG);
            // and also must release super lock
            super.releaseExclusiveReadLockOnAbort(operations, file, exchange);
        }
    }
    // store read-lock state
    exchange.setProperty(asReadLockKey(file, Exchange.FILE_LOCK_EXCLUSIVE_LOCK), lock);
    exchange.setProperty(asReadLockKey(file, Exchange.FILE_LOCK_RANDOM_ACCESS_FILE), randomAccessFile);
    // we grabbed the lock
    return true;
}
Also used : RandomAccessFile(java.io.RandomAccessFile) FileChannel(java.nio.channels.FileChannel) FileLock(java.nio.channels.FileLock) IOException(java.io.IOException) RandomAccessFile(java.io.RandomAccessFile) GenericFile(org.apache.camel.component.file.GenericFile) File(java.io.File) StopWatch(org.apache.camel.util.StopWatch)

Example 2 with FileChannel

use of java.nio.channels.FileChannel in project camel by apache.

the class FileOperations method writeFileByStream.

private void writeFileByStream(InputStream in, File target) throws IOException {
    FileChannel out = null;
    try {
        out = prepareOutputFileChannel(target);
        LOG.debug("Using InputStream to write file: {}", target);
        int size = endpoint.getBufferSize();
        byte[] buffer = new byte[size];
        ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
        int bytesRead;
        while ((bytesRead = in.read(buffer)) != -1) {
            if (bytesRead < size) {
                byteBuffer.limit(bytesRead);
            }
            out.write(byteBuffer);
            byteBuffer.clear();
        }
    } finally {
        IOHelper.close(in, target.getName(), LOG);
        IOHelper.close(out, target.getName(), LOG, endpoint.isForceWrites());
    }
}
Also used : FileChannel(java.nio.channels.FileChannel) ByteBuffer(java.nio.ByteBuffer)

Example 3 with FileChannel

use of java.nio.channels.FileChannel in project camel by apache.

the class FileUtil method copyFile.

/**
     * Copies the file
     *
     * @param from  the source file
     * @param to    the destination file
     * @throws IOException If an I/O error occurs during copy operation
     */
public static void copyFile(File from, File to) throws IOException {
    FileChannel in = null;
    FileChannel out = null;
    try {
        in = new FileInputStream(from).getChannel();
        out = new FileOutputStream(to).getChannel();
        if (LOG.isTraceEnabled()) {
            LOG.trace("Using FileChannel to copy from: " + in + " to: " + out);
        }
        long size = in.size();
        long position = 0;
        while (position < size) {
            position += in.transferTo(position, BUFFER_SIZE, out);
        }
    } finally {
        IOHelper.close(in, from.getName(), LOG);
        IOHelper.close(out, to.getName(), LOG);
    }
}
Also used : FileChannel(java.nio.channels.FileChannel) FileOutputStream(java.io.FileOutputStream) FileInputStream(java.io.FileInputStream)

Example 4 with FileChannel

use of java.nio.channels.FileChannel in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testMultipleSequences.

@Test
public void testMultipleSequences() {
    File secondFile = null;
    FileChannel secondChannel = null;
    try {
        // create the second file channel
        secondFile = File.createTempFile("testdata", "tmp");
        secondChannel = new RandomAccessFile(secondFile, "rw").getChannel();
        final Random rnd = new Random();
        final Random bufferRnd = new Random();
        final long bufferSeed = rnd.nextLong();
        bufferRnd.setSeed(bufferSeed);
        final int numEventsAndBuffers1 = 272;
        final int numEventsAndBuffers2 = 151;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events1 = new ArrayList<BufferOrEvent>(128);
        final ArrayList<BufferOrEvent> events2 = new ArrayList<BufferOrEvent>(128);
        for (int i = 0; i < numEventsAndBuffers1; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events1.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
            } else {
                writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        for (int i = 0; i < numEventsAndBuffers2; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events2.add(generateAndWriteEvent(secondChannel, rnd, numChannels));
            } else {
                writeBuffer(secondChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // reset and create reader
        fileChannel.position(0L);
        secondChannel.position(0L);
        bufferRnd.setSeed(bufferSeed);
        SpilledBufferOrEventSequence seq1 = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        SpilledBufferOrEventSequence seq2 = new SpilledBufferOrEventSequence(secondFile, secondChannel, buffer, pageSize);
        // read and validate the sequence 1
        seq1.open();
        int numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers1; i++) {
            BufferOrEvent next = seq1.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events1.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        assertNull(seq1.getNext());
        assertEquals(events1.size(), numEvent);
        // read and validate the sequence 2
        seq2.open();
        numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers2; i++) {
            BufferOrEvent next = seq2.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events2.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        assertNull(seq2.getNext());
        assertEquals(events2.size(), numEvent);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (secondChannel != null) {
            try {
                secondChannel.close();
            } catch (IOException e) {
            // ignore here
            }
        }
        if (secondFile != null) {
            //noinspection ResultOfMethodCallIgnored
            secondFile.delete();
        }
    }
}
Also used : FileChannel(java.nio.channels.FileChannel) ArrayList(java.util.ArrayList) IOException(java.io.IOException) IOException(java.io.IOException) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) SpilledBufferOrEventSequence(org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence) RandomAccessFile(java.io.RandomAccessFile) Random(java.util.Random) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Test(org.junit.Test)

Example 5 with FileChannel

use of java.nio.channels.FileChannel in project hadoop by apache.

the class FsDatasetTestUtil method assertFileLockReleased.

/**
   * Asserts that the storage lock file in the given directory has been
   * released.  This method works by trying to acquire the lock file itself.  If
   * locking fails here, then the main code must have failed to release it.
   *
   * @param dir the storage directory to check
   * @throws IOException if there is an unexpected I/O error
   */
public static void assertFileLockReleased(String dir) throws IOException {
    StorageLocation sl = StorageLocation.parse(dir);
    File lockFile = new File(new File(sl.getUri()), Storage.STORAGE_FILE_LOCK);
    try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rws");
        FileChannel channel = raf.getChannel()) {
        FileLock lock = channel.tryLock();
        assertNotNull(String.format("Lock file at %s appears to be held by a different process.", lockFile.getAbsolutePath()), lock);
        if (lock != null) {
            try {
                lock.release();
            } catch (IOException e) {
                FsDatasetImpl.LOG.warn(String.format("I/O error releasing file lock %s.", lockFile.getAbsolutePath()), e);
                throw e;
            }
        }
    } catch (OverlappingFileLockException e) {
        fail(String.format("Must release lock file at %s.", lockFile.getAbsolutePath()));
    }
}
Also used : RandomAccessFile(java.io.RandomAccessFile) FileChannel(java.nio.channels.FileChannel) FileLock(java.nio.channels.FileLock) IOException(java.io.IOException) StorageLocation(org.apache.hadoop.hdfs.server.datanode.StorageLocation) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) OverlappingFileLockException(java.nio.channels.OverlappingFileLockException)

Aggregations

FileChannel (java.nio.channels.FileChannel)629 IOException (java.io.IOException)227 ByteBuffer (java.nio.ByteBuffer)205 File (java.io.File)185 FileInputStream (java.io.FileInputStream)164 FileOutputStream (java.io.FileOutputStream)147 RandomAccessFile (java.io.RandomAccessFile)144 Test (org.junit.Test)95 MappedByteBuffer (java.nio.MappedByteBuffer)78 Path (java.nio.file.Path)37 FileLock (java.nio.channels.FileLock)32 FileNotFoundException (java.io.FileNotFoundException)29 Random (java.util.Random)12 OutputStream (java.io.OutputStream)11 ArrayList (java.util.ArrayList)11 AsynchronousFileChannel (java.nio.channels.AsynchronousFileChannel)10 OverlappingFileLockException (java.nio.channels.OverlappingFileLockException)10 LinkedList (java.util.LinkedList)10 ProjectWorkspace (com.facebook.buck.testutil.integration.ProjectWorkspace)9 BufferedReader (java.io.BufferedReader)9