Examples with FileInfo org.apache.nifi.processors.standard.util.FileInfo used on opensource projects

Example 1 with FileInfo

use of org.apache.nifi.processors.standard.util.FileInfo in project nifi by apache.

the class GetFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final long pollingIntervalMillis = context.getProperty(FileTransfer.POLLING_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);
    final long nextPollTime = lastPollTime.get() + pollingIntervalMillis;
    BlockingQueue<FileInfo> fileQueue = fileQueueRef.get();
    final ComponentLog logger = getLogger();
    // do not do the listing if there are already 100 or more items in our queue
    // 100 is really just a magic number that seems to work out well in practice
    FileTransfer transfer = null;
    if (System.currentTimeMillis() >= nextPollTime && (fileQueue == null || fileQueue.size() < 100) && listingLock.tryLock()) {
        try {
            transfer = getFileTransfer(context);
            try {
                fetchListing(context, session, transfer);
                lastPollTime.set(System.currentTimeMillis());
            } catch (final IOException e) {
                context.yield();
                try {
                    transfer.close();
                } catch (final IOException e1) {
                    logger.warn("Unable to close connection due to {}", new Object[] { e1 });
                }
                logger.error("Unable to fetch listing from remote server due to {}", new Object[] { e });
                return;
            }
        } finally {
            listingLock.unlock();
        }
    }
    fileQueue = fileQueueRef.get();
    if (fileQueue == null || fileQueue.isEmpty()) {
        // nothing to do!
        context.yield();
        if (transfer != null) {
            try {
                transfer.close();
            } catch (final IOException e1) {
                logger.warn("Unable to close connection due to {}", new Object[] { e1 });
            }
        }
        return;
    }
    final String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions().getValue();
    final boolean deleteOriginal = context.getProperty(FileTransfer.DELETE_ORIGINAL).asBoolean();
    final int maxSelects = context.getProperty(FileTransfer.MAX_SELECTS).asInteger();
    if (transfer == null) {
        transfer = getFileTransfer(context);
    }
    try {
        for (int i = 0; i < maxSelects && isScheduled(); i++) {
            final FileInfo file;
            sharableTransferLock.lock();
            try {
                file = fileQueue.poll();
                if (file == null) {
                    return;
                }
                processing.add(file);
            } finally {
                sharableTransferLock.unlock();
            }
            File relativeFile = new File(file.getFullPathFileName());
            final String parentRelativePath = (null == relativeFile.getParent()) ? "" : relativeFile.getParent();
            final String parentRelativePathString = parentRelativePath + "/";
            final Path absPath = relativeFile.toPath().toAbsolutePath();
            final String absPathString = absPath.getParent().toString() + "/";
            try {
                FlowFile flowFile = session.create();
                final StopWatch stopWatch = new StopWatch(false);
                try (final InputStream in = transfer.getInputStream(file.getFullPathFileName())) {
                    stopWatch.start();
                    flowFile = session.importFrom(in, flowFile);
                    stopWatch.stop();
                }
                transfer.flush();
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
                flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", hostname);
                flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), parentRelativePathString);
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), relativeFile.getName());
                flowFile = session.putAttribute(flowFile, CoreAttributes.ABSOLUTE_PATH.key(), absPathString);
                Map<String, String> attributes = getAttributesFromFile(file);
                if (attributes.size() > 0) {
                    flowFile = session.putAllAttributes(flowFile, attributes);
                }
                if (deleteOriginal) {
                    try {
                        transfer.deleteFile(flowFile, null, file.getFullPathFileName());
                    } catch (final IOException e) {
                        logger.error("Failed to remove remote file {} due to {}; deleting local copy", new Object[] { file.getFullPathFileName(), e });
                        session.remove(flowFile);
                        return;
                    }
                }
                session.getProvenanceReporter().receive(flowFile, transfer.getProtocolName() + "://" + hostname + "/" + file.getFullPathFileName(), millis);
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully retrieved {} from {} in {} milliseconds at a rate of {} and transferred to success", new Object[] { flowFile, hostname, millis, dataRate });
                session.commit();
            } catch (final IOException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e });
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", new Object[] { e1 });
                }
                session.rollback();
                return;
            } catch (final FlowFileAccessException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e.getCause() }, e);
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", e1);
                }
                session.rollback();
                return;
            } finally {
                processing.remove(file);
            }
        }
    } finally {
        try {
            transfer.close();
        } catch (final IOException e) {
            logger.warn("Failed to close connection to {} due to {}", new Object[] { hostname, e });
        }
    }
}

Example 2 with FileInfo

use of org.apache.nifi.processors.standard.util.FileInfo in project nifi by apache.

the class ListFile method performListing.

@Override
protected List<FileInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    final Path path = new File(getPath(context)).toPath();
    final Boolean recurse = context.getProperty(RECURSE).asBoolean();
    final BiPredicate<Path, BasicFileAttributes> fileFilter = fileFilterRef.get();
    int maxDepth = recurse ? Integer.MAX_VALUE : 1;
    Stream<Path> inputStream = Files.find(path, maxDepth, (p, attributes) -> !attributes.isDirectory() && (minTimestamp == null || attributes.lastModifiedTime().toMillis() >= minTimestamp) && fileFilter.test(p, attributes));
    Stream<FileInfo> listing = inputStream.map(p -> {
        File file = p.toFile();
        return new FileInfo.Builder().directory(file.isDirectory()).filename(file.getName()).fullPathFileName(file.getAbsolutePath()).lastModifiedTime(file.lastModified()).build();
    });
    return listing.collect(Collectors.toList());
}

Example 3 with FileInfo

use of org.apache.nifi.processors.standard.util.FileInfo in project nifi by apache.

the class ListFileTransfer method performListing.

@Override
protected List<FileInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    final FileTransfer transfer = getFileTransfer(context);
    final List<FileInfo> listing;
    try {
        listing = transfer.getListing();
    } finally {
        IOUtils.closeQuietly(transfer);
    }
    if (minTimestamp == null) {
        return listing;
    }
    final Iterator<FileInfo> itr = listing.iterator();
    while (itr.hasNext()) {
        final FileInfo next = itr.next();
        if (next.getLastModifiedTime() < minTimestamp) {
            itr.remove();
        }
    }
    return listing;
}

Example 4 with FileInfo

use of org.apache.nifi.processors.standard.util.FileInfo in project nifi by apache.

the class GetFileTransfer method fetchListing.

// must be called while holding the listingLock
private void fetchListing(final ProcessContext context, final ProcessSession session, final FileTransfer transfer) throws IOException {
    BlockingQueue<FileInfo> queue = fileQueueRef.get();
    if (queue == null) {
        final boolean useNaturalOrdering = context.getProperty(FileTransfer.USE_NATURAL_ORDERING).asBoolean();
        queue = useNaturalOrdering ? new PriorityBlockingQueue<FileInfo>(25000) : new LinkedBlockingQueue<FileInfo>(25000);
        fileQueueRef.set(queue);
    }
    final StopWatch stopWatch = new StopWatch(true);
    final List<FileInfo> listing = transfer.getListing();
    final long millis = stopWatch.getElapsed(TimeUnit.MILLISECONDS);
    int newItems = 0;
    mutuallyExclusiveTransferLock.lock();
    try {
        for (final FileInfo file : listing) {
            if (!queue.contains(file) && !processing.contains(file)) {
                if (!queue.offer(file)) {
                    break;
                }
                newItems++;
            }
        }
    } finally {
        mutuallyExclusiveTransferLock.unlock();
    }
    getLogger().info("Obtained file listing in {} milliseconds; listing had {} items, {} of which were new", new Object[] { millis, listing.size(), newItems });
}

Example 5 with FileInfo

use of org.apache.nifi.processors.standard.util.FileInfo in project nifi by apache.

the class PutFileTransfer method identifyAndResolveConflictFile.

// Attempts to identify naming or content issues with files before they are transferred.
private ConflictResult identifyAndResolveConflictFile(final String conflictResolutionType, final T transfer, final String path, final FlowFile flowFile, final boolean rejectZeroByteFiles, final ComponentLog logger) throws IOException {
    Relationship destinationRelationship = REL_SUCCESS;
    String fileName = flowFile.getAttribute(CoreAttributes.FILENAME.key());
    boolean transferFile = true;
    boolean penalizeFile = false;
    // Reject files that are zero bytes or less
    if (rejectZeroByteFiles) {
        final long sizeInBytes = flowFile.getSize();
        if (sizeInBytes == 0) {
            logger.warn("Rejecting {} because it is zero bytes", new Object[] { flowFile });
            return new ConflictResult(REL_REJECT, false, fileName, true);
        }
    }
    // Second, check if the user doesn't care about detecting naming conflicts ahead of time
    if (conflictResolutionType.equalsIgnoreCase(FileTransfer.CONFLICT_RESOLUTION_NONE)) {
        return new ConflictResult(destinationRelationship, transferFile, fileName, penalizeFile);
    }
    final FileInfo remoteFileInfo = transfer.getRemoteFileInfo(flowFile, path, fileName);
    if (remoteFileInfo == null) {
        return new ConflictResult(destinationRelationship, transferFile, fileName, penalizeFile);
    }
    if (remoteFileInfo.isDirectory()) {
        logger.warn("Resolving conflict by rejecting {} due to conflicting filename with a directory or file already on remote server", new Object[] { flowFile });
        return new ConflictResult(REL_REJECT, false, fileName, false);
    }
    logger.info("Discovered a filename conflict on the remote server for {} so handling using configured Conflict Resolution of {}", new Object[] { flowFile, conflictResolutionType });
    switch(conflictResolutionType.toUpperCase()) {
        case FileTransfer.CONFLICT_RESOLUTION_REJECT:
            destinationRelationship = REL_REJECT;
            transferFile = false;
            penalizeFile = false;
            logger.warn("Resolving conflict by rejecting {} due to conflicting filename with a directory or file already on remote server", new Object[] { flowFile });
            break;
        case FileTransfer.CONFLICT_RESOLUTION_REPLACE:
            transfer.deleteFile(flowFile, path, fileName);
            destinationRelationship = REL_SUCCESS;
            transferFile = true;
            penalizeFile = false;
            logger.info("Resolving filename conflict for {} with remote server by deleting remote file and replacing with flow file", new Object[] { flowFile });
            break;
        case FileTransfer.CONFLICT_RESOLUTION_RENAME:
            boolean uniqueNameGenerated = false;
            for (int i = 1; i < 100 && !uniqueNameGenerated; i++) {
                String possibleFileName = i + "." + fileName;
                final FileInfo renamedFileInfo = transfer.getRemoteFileInfo(flowFile, path, possibleFileName);
                uniqueNameGenerated = (renamedFileInfo == null);
                if (uniqueNameGenerated) {
                    fileName = possibleFileName;
                    logger.info("Attempting to resolve filename conflict for {} on the remote server by using a newly generated filename of: {}", new Object[] { flowFile, fileName });
                    destinationRelationship = REL_SUCCESS;
                    transferFile = true;
                    penalizeFile = false;
                    break;
                }
            }
            if (!uniqueNameGenerated) {
                destinationRelationship = REL_REJECT;
                transferFile = false;
                penalizeFile = false;
                logger.warn("Could not determine a unique name after 99 attempts for.  Switching resolution mode to REJECT for " + flowFile);
            }
            break;
        case FileTransfer.CONFLICT_RESOLUTION_IGNORE:
            destinationRelationship = REL_SUCCESS;
            transferFile = false;
            penalizeFile = false;
            logger.info("Resolving conflict for {}  by not transferring file and and still considering the process a success.", new Object[] { flowFile });
            break;
        case FileTransfer.CONFLICT_RESOLUTION_FAIL:
            destinationRelationship = REL_FAILURE;
            transferFile = false;
            penalizeFile = true;
            logger.warn("Resolved filename conflict for {} as configured by routing to FAILURE relationship.", new Object[] { flowFile });
        default:
            break;
    }
    return new ConflictResult(destinationRelationship, transferFile, fileName, penalizeFile);
}