Examples with FTPListParseEngine org.apache.commons.net.ftp.FTPListParseEngine used on opensource projects

Example 1 with FTPListParseEngine

use of org.apache.commons.net.ftp.FTPListParseEngine in project onebusaway-application-modules by camsys.

the class OrbcadRecordFtpSource method getUpdatedFilesToDownload.

private List<String> getUpdatedFilesToDownload() throws IOException {
    long t1 = SystemTime.currentTimeMillis();
    FTPListParseEngine engine = _ftpClient.initiateListParsing(_dataDirectory);
    Set<String> paths = new HashSet<String>();
    List<String> toDownload = new ArrayList<String>();
    while (engine.hasNext()) {
        // "page size" you want
        FTPFile[] files = engine.getNext(25);
        for (FTPFile file : files) {
            String path = _dataDirectory + "/" + file.getName();
            paths.add(path);
            if (!_paths.contains(path))
                toDownload.add(path);
        }
    }
    _totalFtpFiles = paths.size();
    _newFtpFiles = toDownload.size();
    long t2 = SystemTime.currentTimeMillis();
    if (_log.isDebugEnabled())
        _log.debug("file listing time: " + (t2 - t1) + " totalFiles: " + paths.size() + " newFiles: " + toDownload.size());
    _paths = paths;
    if (_maxDownloadCount > 0 && toDownload.size() > _maxDownloadCount) {
        List<String> reduced = new ArrayList<String>(_maxDownloadCount);
        for (int i = 0; i < _maxDownloadCount; i++) reduced.add(toDownload.get(toDownload.size() - _maxDownloadCount + i));
        toDownload = reduced;
    }
    return toDownload;
}