Examples with FTPFile org.apache.commons.net.ftp.FTPFile used on opensource projects

Example 56 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project camel by apache.

the class FtpOperations method retrieveFileToFileInLocalWorkDirectory.

@SuppressWarnings("unchecked")
private boolean retrieveFileToFileInLocalWorkDirectory(String name, Exchange exchange) throws GenericFileOperationFailedException {
    File temp;
    File local = new File(FileUtil.normalizePath(endpoint.getLocalWorkDirectory()));
    OutputStream os;
    try {
        // use relative filename in local work directory
        GenericFile<FTPFile> target = (GenericFile<FTPFile>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
        ObjectHelper.notNull(target, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");
        String relativeName = target.getRelativeFilePath();
        temp = new File(local, relativeName + ".inprogress");
        local = new File(local, relativeName);
        // create directory to local work file
        local.mkdirs();
        // delete any existing files
        if (temp.exists()) {
            if (!FileUtil.deleteFile(temp)) {
                throw new GenericFileOperationFailedException("Cannot delete existing local work file: " + temp);
            }
        }
        if (local.exists()) {
            if (!FileUtil.deleteFile(local)) {
                throw new GenericFileOperationFailedException("Cannot delete existing local work file: " + local);
            }
        }
        // create new temp local work file
        if (!temp.createNewFile()) {
            throw new GenericFileOperationFailedException("Cannot create new local work file: " + temp);
        }
        // store content as a file in the local work directory in the temp handle
        os = new FileOutputStream(temp);
        // set header with the path to the local work file            
        exchange.getIn().setHeader(Exchange.FILE_LOCAL_WORK_PATH, local.getPath());
    } catch (Exception e) {
        throw new GenericFileOperationFailedException("Cannot create new local work file: " + local);
    }
    boolean result;
    try {
        GenericFile<FTPFile> target = (GenericFile<FTPFile>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
        // store the java.io.File handle as the body
        target.setBody(local);
        String remoteName = name;
        String currentDir = null;
        if (endpoint.getConfiguration().isStepwise()) {
            // remember current directory
            currentDir = getCurrentDirectory();
            // change directory to path where the file is to be retrieved
            // (must do this as some FTP servers cannot retrieve using absolute path)
            String path = FileUtil.onlyPath(name);
            if (path != null) {
                changeCurrentDirectory(path);
            }
            // remote name is now only the file name as we just changed directory
            remoteName = FileUtil.stripPath(name);
        }
        log.trace("Client retrieveFile: {}", remoteName);
        result = client.retrieveFile(remoteName, os);
        // store client reply information after the operation
        exchange.getIn().setHeader(FtpConstants.FTP_REPLY_CODE, client.getReplyCode());
        exchange.getIn().setHeader(FtpConstants.FTP_REPLY_STRING, client.getReplyString());
        // change back to current directory
        if (endpoint.getConfiguration().isStepwise()) {
            changeCurrentDirectory(currentDir);
        }
    } catch (IOException e) {
        log.trace("Error occurred during retrieving file: {} to local directory. Deleting local work file: {}", name, temp);
        // failed to retrieve the file so we need to close streams and delete in progress file
        // must close stream before deleting file
        IOHelper.close(os, "retrieve: " + name, log);
        boolean deleted = FileUtil.deleteFile(temp);
        if (!deleted) {
            log.warn("Error occurred during retrieving file: " + name + " to local directory. Cannot delete local work file: " + temp);
        }
        throw new GenericFileOperationFailedException(client.getReplyCode(), client.getReplyString(), e.getMessage(), e);
    } finally {
        // need to close the stream before rename it
        IOHelper.close(os, "retrieve: " + name, log);
    }
    log.debug("Retrieve file to local work file result: {}", result);
    if (result) {
        log.trace("Renaming local in progress file from: {} to: {}", temp, local);
        // operation went okay so rename temp to local after we have retrieved the data
        try {
            if (!FileUtil.renameFile(temp, local, false)) {
                throw new GenericFileOperationFailedException("Cannot rename local work file from: " + temp + " to: " + local);
            }
        } catch (IOException e) {
            throw new GenericFileOperationFailedException("Cannot rename local work file from: " + temp + " to: " + local, e);
        }
    }
    return result;
}

Example 57 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project camel by apache.

the class FtpChangedExclusiveReadLockStrategy method acquireExclusiveReadLock.

public boolean acquireExclusiveReadLock(GenericFileOperations<FTPFile> operations, GenericFile<FTPFile> file, Exchange exchange) throws Exception {
    boolean exclusive = false;
    LOG.trace("Waiting for exclusive read lock to file: " + file);
    long lastModified = Long.MIN_VALUE;
    long length = Long.MIN_VALUE;
    StopWatch watch = new StopWatch();
    long startTime = new Date().getTime();
    while (!exclusive) {
        // timeout check
        if (timeout > 0) {
            long delta = watch.taken();
            if (delta > timeout) {
                CamelLogger.log(LOG, readLockLoggingLevel, "Cannot acquire read lock within " + timeout + " millis. Will skip the file: " + file);
                // we could not get the lock within the timeout period, so return false
                return false;
            }
        }
        long newLastModified = 0;
        long newLength = 0;
        List<FTPFile> files;
        if (fastExistsCheck) {
            // use the absolute file path to only pickup the file we want to check, this avoids expensive
            // list operations if we have a lot of files in the directory
            String path = file.getAbsoluteFilePath();
            if (path.equals("/") || path.equals("\\")) {
                // special for root (= home) directory
                LOG.trace("Using fast exists to update file information in home directory");
                files = operations.listFiles();
            } else {
                LOG.trace("Using fast exists to update file information for {}", path);
                files = operations.listFiles(path);
            }
        } else {
            // fast option not enabled, so list the directory and filter the file name
            String path = file.getParent();
            if (path.equals("/") || path.equals("\\")) {
                // special for root (= home) directory
                LOG.trace("Using full directory listing in home directory to update file information. Consider enabling fastExistsCheck option.");
                files = operations.listFiles();
            } else {
                LOG.trace("Using full directory listing to update file information for {}. Consider enabling fastExistsCheck option.", path);
                files = operations.listFiles(path);
            }
        }
        LOG.trace("List files {} found {} files", file.getAbsoluteFilePath(), files.size());
        for (FTPFile f : files) {
            boolean match;
            if (fastExistsCheck) {
                // uses the absolute file path as well
                match = f.getName().equals(file.getAbsoluteFilePath()) || f.getName().equals(file.getFileNameOnly());
            } else {
                match = f.getName().equals(file.getFileNameOnly());
            }
            if (match) {
                newLength = f.getSize();
                if (f.getTimestamp() != null) {
                    newLastModified = f.getTimestamp().getTimeInMillis();
                }
            }
        }
        LOG.trace("Previous last modified: " + lastModified + ", new last modified: " + newLastModified);
        LOG.trace("Previous length: " + length + ", new length: " + newLength);
        long newOlderThan = startTime + watch.taken() - minAge;
        LOG.trace("New older than threshold: {}", newOlderThan);
        if (newLength >= minLength && ((minAge == 0 && newLastModified == lastModified && newLength == length) || (minAge != 0 && newLastModified < newOlderThan))) {
            LOG.trace("Read lock acquired.");
            exclusive = true;
        } else {
            // set new base file change information
            lastModified = newLastModified;
            length = newLength;
            boolean interrupted = sleep();
            if (interrupted) {
                // we were interrupted while sleeping, we are likely being shutdown so return false
                return false;
            }
        }
    }
    return exclusive;
}

Example 58 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project camel by apache.

the class FtpProcessStrategyFactory method createGenericFileProcessStrategy.

public static GenericFileProcessStrategy<FTPFile> createGenericFileProcessStrategy(CamelContext context, Map<String, Object> params) {
    // We assume a value is present only if its value not null for String and 'true' for boolean
    Expression moveExpression = (Expression) params.get("move");
    Expression moveFailedExpression = (Expression) params.get("moveFailed");
    Expression preMoveExpression = (Expression) params.get("preMove");
    boolean isNoop = params.get("noop") != null;
    boolean isDelete = params.get("delete") != null;
    boolean isMove = moveExpression != null || preMoveExpression != null || moveFailedExpression != null;
    if (isDelete) {
        GenericFileDeleteProcessStrategy<FTPFile> strategy = new GenericFileDeleteProcessStrategy<FTPFile>();
        strategy.setExclusiveReadLockStrategy(getExclusiveReadLockStrategy(params));
        if (preMoveExpression != null) {
            GenericFileExpressionRenamer<FTPFile> renamer = new GenericFileExpressionRenamer<FTPFile>();
            renamer.setExpression(preMoveExpression);
            strategy.setBeginRenamer(renamer);
        }
        if (moveFailedExpression != null) {
            GenericFileExpressionRenamer<FTPFile> renamer = new GenericFileExpressionRenamer<FTPFile>();
            renamer.setExpression(moveFailedExpression);
            strategy.setFailureRenamer(renamer);
        }
        return strategy;
    } else if (isMove || isNoop) {
        GenericFileRenameProcessStrategy<FTPFile> strategy = new GenericFileRenameProcessStrategy<FTPFile>();
        strategy.setExclusiveReadLockStrategy(getExclusiveReadLockStrategy(params));
        if (!isNoop && moveExpression != null) {
            // move on commit is only possible if not noop
            GenericFileExpressionRenamer<FTPFile> renamer = new GenericFileExpressionRenamer<FTPFile>();
            renamer.setExpression(moveExpression);
            strategy.setCommitRenamer(renamer);
        }
        // both move and noop supports pre move
        if (moveFailedExpression != null) {
            GenericFileExpressionRenamer<FTPFile> renamer = new GenericFileExpressionRenamer<FTPFile>();
            renamer.setExpression(moveFailedExpression);
            strategy.setFailureRenamer(renamer);
        }
        // both move and noop supports pre move
        if (preMoveExpression != null) {
            GenericFileExpressionRenamer<FTPFile> renamer = new GenericFileExpressionRenamer<FTPFile>();
            renamer.setExpression(preMoveExpression);
            strategy.setBeginRenamer(renamer);
        }
        return strategy;
    } else {
        // default strategy will do nothing
        GenericFileNoOpProcessStrategy<FTPFile> strategy = new GenericFileNoOpProcessStrategy<FTPFile>();
        strategy.setExclusiveReadLockStrategy(getExclusiveReadLockStrategy(params));
        return strategy;
    }
}

Example 59 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project nifi by apache.

the class FTPTransfer method newFileInfo.

private FileInfo newFileInfo(final FTPFile file, String path) {
    if (file == null) {
        return null;
    }
    final File newFullPath = new File(path, file.getName());
    final String newFullForwardPath = newFullPath.getPath().replace("\\", "/");
    StringBuilder perms = new StringBuilder();
    perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-");
    perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-");
    perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-");
    perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-");
    perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-");
    perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-");
    perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-");
    perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-");
    perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-");
    FileInfo.Builder builder = new FileInfo.Builder().filename(file.getName()).fullPathFileName(newFullForwardPath).directory(file.isDirectory()).size(file.getSize()).lastModifiedTime(file.getTimestamp().getTimeInMillis()).permissions(perms.toString()).owner(file.getUser()).group(file.getGroup());
    return builder.build();
}

Example 60 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project nifi by apache.

the class FTPTransfer method getListing.

private List<FileInfo> getListing(final String path, final int depth, final int maxResults) throws IOException {
    final List<FileInfo> listing = new ArrayList<>();
    if (maxResults < 1) {
        return listing;
    }
    if (depth >= 100) {
        logger.warn(this + " had to stop recursively searching directories at a recursive depth of " + depth + " to avoid memory issues");
        return listing;
    }
    final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean();
    final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
    final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
    final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
    final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
    final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex);
    final String remotePath = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions().getValue();
    // check if this directory path matches the PATH_FILTER_REGEX
    boolean pathFilterMatches = true;
    if (pathPattern != null) {
        Path reldir = path == null ? Paths.get(".") : Paths.get(path);
        if (remotePath != null) {
            reldir = Paths.get(remotePath).relativize(reldir);
        }
        if (reldir != null && !reldir.toString().isEmpty()) {
            if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) {
                pathFilterMatches = false;
            }
        }
    }
    final FTPClient client = getClient(null);
    int count = 0;
    final FTPFile[] files;
    if (path == null || path.trim().isEmpty()) {
        files = client.listFiles(".");
    } else {
        files = client.listFiles(path);
    }
    if (files.length == 0 && path != null && !path.trim().isEmpty()) {
        // throw exception if directory doesn't exist
        final boolean cdSuccessful = setWorkingDirectory(path);
        if (!cdSuccessful) {
            throw new IOException("Cannot list files for non-existent directory " + path);
        }
    }
    for (final FTPFile file : files) {
        final String filename = file.getName();
        if (filename.equals(".") || filename.equals("..")) {
            continue;
        }
        if (ignoreDottedFiles && filename.startsWith(".")) {
            continue;
        }
        final File newFullPath = new File(path, filename);
        final String newFullForwardPath = newFullPath.getPath().replace("\\", "/");
        if (recurse && file.isDirectory()) {
            try {
                listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count));
            } catch (final IOException e) {
                logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e);
            }
        }
        // FILE_FILTER_REGEX - then let's add it
        if (!file.isDirectory() && !file.isSymbolicLink() && pathFilterMatches) {
            if (pattern == null || pattern.matcher(filename).matches()) {
                listing.add(newFileInfo(file, path));
                count++;
            }
        }
        if (count >= maxResults) {
            break;
        }
    }
    return listing;
}