Examples with FileTransfer org.apache.nifi.processors.standard.util.FileTransfer used on opensource projects

Example 1 with FileTransfer

use of org.apache.nifi.processors.standard.util.FileTransfer in project nifi by apache.

the class GetFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final long pollingIntervalMillis = context.getProperty(FileTransfer.POLLING_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);
    final long nextPollTime = lastPollTime.get() + pollingIntervalMillis;
    BlockingQueue<FileInfo> fileQueue = fileQueueRef.get();
    final ComponentLog logger = getLogger();
    // do not do the listing if there are already 100 or more items in our queue
    // 100 is really just a magic number that seems to work out well in practice
    FileTransfer transfer = null;
    if (System.currentTimeMillis() >= nextPollTime && (fileQueue == null || fileQueue.size() < 100) && listingLock.tryLock()) {
        try {
            transfer = getFileTransfer(context);
            try {
                fetchListing(context, session, transfer);
                lastPollTime.set(System.currentTimeMillis());
            } catch (final IOException e) {
                context.yield();
                try {
                    transfer.close();
                } catch (final IOException e1) {
                    logger.warn("Unable to close connection due to {}", new Object[] { e1 });
                }
                logger.error("Unable to fetch listing from remote server due to {}", new Object[] { e });
                return;
            }
        } finally {
            listingLock.unlock();
        }
    }
    fileQueue = fileQueueRef.get();
    if (fileQueue == null || fileQueue.isEmpty()) {
        // nothing to do!
        context.yield();
        if (transfer != null) {
            try {
                transfer.close();
            } catch (final IOException e1) {
                logger.warn("Unable to close connection due to {}", new Object[] { e1 });
            }
        }
        return;
    }
    final String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions().getValue();
    final boolean deleteOriginal = context.getProperty(FileTransfer.DELETE_ORIGINAL).asBoolean();
    final int maxSelects = context.getProperty(FileTransfer.MAX_SELECTS).asInteger();
    if (transfer == null) {
        transfer = getFileTransfer(context);
    }
    try {
        for (int i = 0; i < maxSelects && isScheduled(); i++) {
            final FileInfo file;
            sharableTransferLock.lock();
            try {
                file = fileQueue.poll();
                if (file == null) {
                    return;
                }
                processing.add(file);
            } finally {
                sharableTransferLock.unlock();
            }
            File relativeFile = new File(file.getFullPathFileName());
            final String parentRelativePath = (null == relativeFile.getParent()) ? "" : relativeFile.getParent();
            final String parentRelativePathString = parentRelativePath + "/";
            final Path absPath = relativeFile.toPath().toAbsolutePath();
            final String absPathString = absPath.getParent().toString() + "/";
            try {
                FlowFile flowFile = session.create();
                final StopWatch stopWatch = new StopWatch(false);
                try (final InputStream in = transfer.getInputStream(file.getFullPathFileName())) {
                    stopWatch.start();
                    flowFile = session.importFrom(in, flowFile);
                    stopWatch.stop();
                }
                transfer.flush();
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
                flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", hostname);
                flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), parentRelativePathString);
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), relativeFile.getName());
                flowFile = session.putAttribute(flowFile, CoreAttributes.ABSOLUTE_PATH.key(), absPathString);
                Map<String, String> attributes = getAttributesFromFile(file);
                if (attributes.size() > 0) {
                    flowFile = session.putAllAttributes(flowFile, attributes);
                }
                if (deleteOriginal) {
                    try {
                        transfer.deleteFile(flowFile, null, file.getFullPathFileName());
                    } catch (final IOException e) {
                        logger.error("Failed to remove remote file {} due to {}; deleting local copy", new Object[] { file.getFullPathFileName(), e });
                        session.remove(flowFile);
                        return;
                    }
                }
                session.getProvenanceReporter().receive(flowFile, transfer.getProtocolName() + "://" + hostname + "/" + file.getFullPathFileName(), millis);
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully retrieved {} from {} in {} milliseconds at a rate of {} and transferred to success", new Object[] { flowFile, hostname, millis, dataRate });
                session.commit();
            } catch (final IOException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e });
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", new Object[] { e1 });
                }
                session.rollback();
                return;
            } catch (final FlowFileAccessException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e.getCause() }, e);
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", e1);
                }
                session.rollback();
                return;
            } finally {
                processing.remove(file);
            }
        }
    } finally {
        try {
            transfer.close();
        } catch (final IOException e) {
            logger.warn("Failed to close connection to {} due to {}", new Object[] { hostname, e });
        }
    }
}

Example 2 with FileTransfer

use of org.apache.nifi.processors.standard.util.FileTransfer in project nifi by apache.

the class ListFileTransfer method performListing.

@Override
protected List<FileInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    final FileTransfer transfer = getFileTransfer(context);
    final List<FileInfo> listing;
    try {
        listing = transfer.getListing();
    } finally {
        IOUtils.closeQuietly(transfer);
    }
    if (minTimestamp == null) {
        return listing;
    }
    final Iterator<FileInfo> itr = listing.iterator();
    while (itr.hasNext()) {
        final FileInfo next = itr.next();
        if (next.getLastModifiedTime() < minTimestamp) {
            itr.remove();
        }
    }
    return listing;
}

Example 3 with FileTransfer

use of org.apache.nifi.processors.standard.util.FileTransfer in project nifi by apache.

the class FetchFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final String host = context.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
    final int port = context.getProperty(UNDEFAULTED_PORT).evaluateAttributeExpressions(flowFile).asInteger();
    final String filename = context.getProperty(REMOTE_FILENAME).evaluateAttributeExpressions(flowFile).getValue();
    // Try to get a FileTransfer object from our cache.
    BlockingQueue<FileTransferIdleWrapper> transferQueue;
    synchronized (fileTransferMap) {
        final Tuple<String, Integer> tuple = new Tuple<>(host, port);
        transferQueue = fileTransferMap.get(tuple);
        if (transferQueue == null) {
            transferQueue = new LinkedBlockingQueue<>();
            fileTransferMap.put(tuple, transferQueue);
        }
        // periodically close idle connections
        if (System.currentTimeMillis() - lastClearTime > IDLE_CONNECTION_MILLIS) {
            closeConnections(false);
            lastClearTime = System.currentTimeMillis();
        }
    }
    // we have a queue of FileTransfer Objects. Get one from the queue or create a new one.
    FileTransfer transfer;
    FileTransferIdleWrapper transferWrapper = transferQueue.poll();
    if (transferWrapper == null) {
        transfer = createFileTransfer(context);
    } else {
        transfer = transferWrapper.getFileTransfer();
    }
    boolean closeConnection = false;
    try {
        // Pull data from remote system.
        final InputStream in;
        try {
            in = transfer.getInputStream(filename, flowFile);
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream out) throws IOException {
                    StreamUtils.copy(in, out);
                }
            });
            if (!transfer.flush(flowFile)) {
                throw new IOException("completePendingCommand returned false, file transfer failed");
            }
        } catch (final FileNotFoundException e) {
            closeConnection = false;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {} because the file could not be found on the remote system; routing to {}", new Object[] { flowFile, filename, host, REL_NOT_FOUND.getName() });
            session.transfer(session.penalize(flowFile), REL_NOT_FOUND);
            session.getProvenanceReporter().route(flowFile, REL_NOT_FOUND);
            return;
        } catch (final PermissionDeniedException e) {
            closeConnection = false;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {} due to insufficient permissions; routing to {}", new Object[] { flowFile, filename, host, REL_PERMISSION_DENIED.getName() });
            session.transfer(session.penalize(flowFile), REL_PERMISSION_DENIED);
            session.getProvenanceReporter().route(flowFile, REL_PERMISSION_DENIED);
            return;
        } catch (final ProcessException | IOException e) {
            closeConnection = true;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {}:{} due to {}; routing to comms.failure", new Object[] { flowFile, filename, host, port, e.toString() }, e);
            session.transfer(session.penalize(flowFile), REL_COMMS_FAILURE);
            return;
        }
        // Add FlowFile attributes
        final String protocolName = transfer.getProtocolName();
        final Map<String, String> attributes = new HashMap<>();
        attributes.put(protocolName + ".remote.host", host);
        attributes.put(protocolName + ".remote.port", String.valueOf(port));
        attributes.put(protocolName + ".remote.filename", filename);
        if (filename.contains("/")) {
            final String path = StringUtils.substringBeforeLast(filename, "/");
            final String filenameOnly = StringUtils.substringAfterLast(filename, "/");
            attributes.put(CoreAttributes.PATH.key(), path);
            attributes.put(CoreAttributes.FILENAME.key(), filenameOnly);
        } else {
            attributes.put(CoreAttributes.FILENAME.key(), filename);
        }
        flowFile = session.putAllAttributes(flowFile, attributes);
        // emit provenance event and transfer FlowFile
        session.getProvenanceReporter().fetch(flowFile, protocolName + "://" + host + ":" + port + "/" + filename, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
        // it is critical that we commit the session before moving/deleting the remote file. Otherwise, we could have a situation where
        // we ingest the data, delete/move the remote file, and then NiFi dies/is shut down before the session is committed. This would
        // result in data loss! If we commit the session first, we are safe.
        session.commit();
        final String completionStrategy = context.getProperty(COMPLETION_STRATEGY).getValue();
        if (COMPLETION_DELETE.getValue().equalsIgnoreCase(completionStrategy)) {
            try {
                transfer.deleteFile(flowFile, null, filename);
            } catch (final FileNotFoundException e) {
            // file doesn't exist -- effectively the same as removing it. Move on.
            } catch (final IOException ioe) {
                getLogger().warn("Successfully fetched the content for {} from {}:{}{} but failed to remove the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe);
            }
        } else if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) {
            String targetDir = context.getProperty(MOVE_DESTINATION_DIR).evaluateAttributeExpressions(flowFile).getValue();
            if (!targetDir.endsWith("/")) {
                targetDir = targetDir + "/";
            }
            final String simpleFilename = StringUtils.substringAfterLast(filename, "/");
            final String target = targetDir + simpleFilename;
            try {
                transfer.rename(flowFile, filename, target);
            } catch (final IOException ioe) {
                getLogger().warn("Successfully fetched the content for {} from {}:{}{} but failed to rename the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe);
            }
        }
    } finally {
        if (transfer != null) {
            if (closeConnection) {
                getLogger().debug("Closing FileTransfer...");
                try {
                    transfer.close();
                } catch (final IOException e) {
                    getLogger().warn("Failed to close connection to {}:{} due to {}", new Object[] { host, port, e.getMessage() }, e);
                }
            } else {
                getLogger().debug("Returning FileTransfer to pool...");
                transferQueue.offer(new FileTransferIdleWrapper(transfer, System.nanoTime()));
            }
        }
    }
}