Examples with FTPFile org.apache.commons.net.ftp.FTPFile used on opensource projects

Example 6 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project bamboobsc by billchen198318.

the class FtpClientUtils method get.

/**
	 * 把目錄下的檔案抓下來,如果有權限
	 * 
	 * @param cwdDirectory		cd 目錄
	 * @param storeDir			本地端目錄
	 * @param head				檔案開頭 代null或 空白 忽略
	 * @param deleteFtpFile		是否刪除 FTP 上檔案(取完後刪除)
	 * 
	 * @throws SocketException
	 * @throws IOException
	 * @throws Exception
	 */
public void get(String cwdDirectory, File storeDir, String head, boolean deleteFtpFile) throws SocketException, IOException, Exception {
    if (!this.ftpClient.isConnected()) {
        this.logger.error("FTP not connection...");
        throw new Exception("FTP not connection...");
    }
    // 非 binary mode 在類 vsFtpd 可能會有問題 
    this.ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
    if (cwdDirectory != null && !"".equals(cwdDirectory)) {
        this.ftpClient.cwd(cwdDirectory);
    }
    FTPFile[] ftpFiles = this.ftpClient.listFiles();
    for (int ix = 0; ftpFiles != null && ix < ftpFiles.length; ix++) {
        if (head != null && !"".equals(head)) {
            if (ftpFiles[ix].getName().indexOf(head) != 0) {
                logger.info("not get : " + ftpFiles[ix].getName());
                continue;
            }
        }
        logger.info(ftpFiles[ix]);
        if (ftpFiles[ix].isFile()) {
            File downloadFile = new File(storeDir.getPath() + "/" + ftpFiles[ix].getName());
            FileOutputStream fos = new FileOutputStream(downloadFile);
            if (this.ftpClient.retrieveFile(ftpFiles[ix].getName(), fos)) {
                logger.info("ftp GET (save to) : " + storeDir.getPath() + "/" + ftpFiles[ix].getName());
                if (deleteFtpFile) {
                    this.delete(ftpFiles[ix].getName());
                }
            }
            downloadFile = null;
            fos.close();
            fos = null;
        }
    }
}

Example 7 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project hive by apache.

the class Ftp method run.

/**
   * Run a thread to transfer files
   */
public void run() {
    byte[] data = null;
    Timer timer = new Timer();
    FTPClient ftp = this.ftp;
    if (currentThreadCnt.getAndIncrement() > 0) {
        ftp = openConnection(null);
    }
    while (true) {
        String file = filesQueue.poll();
        if (file == null) {
            break;
        }
        int num = currentFileCnt.getAndIncrement();
        FTPFile ftpFile = filesMap.get(file);
        long ftpSizeInBytes = ftpFile.getSize();
        String fmtSizeInBytes = Utils.formatSizeInBytes(ftpSizeInBytes);
        String targetFile = getTargetFileName(file);
        if (info) {
            info(null, "  " + file + " - started (" + num + " of " + fileCnt + ", " + fmtSizeInBytes + ")");
        }
        try {
            InputStream in = ftp.retrieveFileStream(file);
            OutputStream out = null;
            java.io.File targetLocalFile = null;
            File targetHdfsFile = null;
            if (local) {
                targetLocalFile = new java.io.File(targetFile);
                if (!targetLocalFile.exists()) {
                    targetLocalFile.getParentFile().mkdirs();
                    targetLocalFile.createNewFile();
                }
                out = new FileOutputStream(targetLocalFile, false);
            } else {
                targetHdfsFile = new File();
                out = targetHdfsFile.create(targetFile, true);
            }
            if (data == null) {
                data = new byte[3 * 1024 * 1024];
            }
            int bytesRead = -1;
            long bytesReadAll = 0;
            long start = timer.start();
            long prev = start;
            long readTime = 0;
            long writeTime = 0;
            long cur, cur2, cur3;
            while (true) {
                cur = timer.current();
                bytesRead = in.read(data);
                cur2 = timer.current();
                readTime += (cur2 - cur);
                if (bytesRead == -1) {
                    break;
                }
                out.write(data, 0, bytesRead);
                out.flush();
                cur3 = timer.current();
                writeTime += (cur3 - cur2);
                bytesReadAll += bytesRead;
                if (info) {
                    cur = timer.current();
                    if (cur - prev > 13000) {
                        long elapsed = cur - start;
                        info(null, "  " + file + " - in progress (" + Utils.formatSizeInBytes(bytesReadAll) + " of " + fmtSizeInBytes + ", " + Utils.formatPercent(bytesReadAll, ftpSizeInBytes) + ", " + Utils.formatTime(elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, readTime) + " read, " + Utils.formatBytesPerSec(bytesReadAll, writeTime) + " write)");
                        prev = cur;
                    }
                }
            }
            if (ftp.completePendingCommand()) {
                in.close();
                cur = timer.current();
                out.close();
                readTime += (timer.current() - cur);
                bytesTransferredAll.addAndGet(bytesReadAll);
                fileCntSuccess.incrementAndGet();
                if (info) {
                    long elapsed = timer.stop();
                    info(null, "  " + file + " - complete (" + Utils.formatSizeInBytes(bytesReadAll) + ", " + Utils.formatTime(elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, elapsed) + ", " + Utils.formatBytesPerSec(bytesReadAll, readTime) + " read, " + Utils.formatBytesPerSec(bytesReadAll, writeTime) + " write)");
                }
            } else {
                in.close();
                out.close();
                if (info) {
                    info(null, "  " + file + " - failed");
                }
                exec.signal(Signal.Type.SQLEXCEPTION, "File transfer failed: " + file);
            }
        } catch (IOException e) {
            exec.signal(e);
        }
    }
    try {
        if (ftp.isConnected()) {
            ftp.logout();
            ftp.disconnect();
        }
    } catch (IOException e) {
    }
}

Example 8 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project hive by apache.

the class Ftp method retrieveFileList.

/**
   * Get the list of files to transfer
   */
void retrieveFileList(String dir) {
    if (info) {
        if (dir == null || dir.isEmpty()) {
            info(null, "  Listing the current working FTP directory");
        } else {
            info(null, "  Listing " + dir);
        }
    }
    try {
        FTPFile[] files = ftp.listFiles(dir);
        ArrayList<FTPFile> dirs = new ArrayList<FTPFile>();
        for (FTPFile file : files) {
            String name = file.getName();
            if (file.isFile()) {
                if (filePattern == null || Pattern.matches(filePattern, name)) {
                    if (dir != null && !dir.isEmpty()) {
                        if (dir.endsWith("/")) {
                            name = dir + name;
                        } else {
                            name = dir + "/" + name;
                        }
                    }
                    if (!newOnly || !isTargetExists(name)) {
                        fileCnt++;
                        ftpSizeInBytes += file.getSize();
                        filesQueue.add(name);
                        filesMap.put(name, file);
                    }
                }
            } else {
                if (subdir && !name.equals(".") && !name.equals("..")) {
                    dirCnt++;
                    dirs.add(file);
                }
            }
        }
        if (subdir) {
            for (FTPFile d : dirs) {
                String sd = d.getName();
                if (dir != null && !dir.isEmpty()) {
                    if (dir.endsWith("/")) {
                        sd = dir + sd;
                    } else {
                        sd = dir + "/" + sd;
                    }
                }
                retrieveFileList(sd);
            }
        }
    } catch (IOException e) {
        exec.signal(e);
    }
}

Example 9 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project hadoop by apache.

the class FTPFileSystem method getFileStatus.

/**
   * Convenience method, so that we don't open a new connection when using this
   * method from within another method. Otherwise every API invocation incurs
   * the overhead of opening/closing a TCP connection.
   */
private FileStatus getFileStatus(FTPClient client, Path file) throws IOException {
    FileStatus fileStat = null;
    Path workDir = new Path(client.printWorkingDirectory());
    Path absolute = makeAbsolute(workDir, file);
    Path parentPath = absolute.getParent();
    if (parentPath == null) {
        // root dir
        // Length of root dir on server not known
        long length = -1;
        boolean isDir = true;
        int blockReplication = 1;
        // Block Size not known.
        long blockSize = DEFAULT_BLOCK_SIZE;
        // Modification time of root dir not known.
        long modTime = -1;
        Path root = new Path("/");
        return new FileStatus(length, isDir, blockReplication, blockSize, modTime, root.makeQualified(this));
    }
    String pathName = parentPath.toUri().getPath();
    FTPFile[] ftpFiles = client.listFiles(pathName);
    if (ftpFiles != null) {
        for (FTPFile ftpFile : ftpFiles) {
            if (ftpFile.getName().equals(file.getName())) {
                // file found in dir
                fileStat = getFileStatus(ftpFile, parentPath);
                break;
            }
        }
        if (fileStat == null) {
            throw new FileNotFoundException("File " + file + " does not exist.");
        }
    } else {
        throw new FileNotFoundException("File " + file + " does not exist.");
    }
    return fileStat;
}

Example 10 with FTPFile

use of org.apache.commons.net.ftp.FTPFile in project camel by apache.

the class FtpOperations method retrieveFileToStreamInBody.

@SuppressWarnings("unchecked")
private boolean retrieveFileToStreamInBody(String name, Exchange exchange) throws GenericFileOperationFailedException {
    boolean result;
    try {
        GenericFile<FTPFile> target = (GenericFile<FTPFile>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
        ObjectHelper.notNull(target, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");
        String remoteName = name;
        String currentDir = null;
        if (endpoint.getConfiguration().isStepwise()) {
            // remember current directory
            currentDir = getCurrentDirectory();
            // change directory to path where the file is to be retrieved
            // (must do this as some FTP servers cannot retrieve using absolute path)
            String path = FileUtil.onlyPath(name);
            if (path != null) {
                changeCurrentDirectory(path);
            }
            // remote name is now only the file name as we just changed directory
            remoteName = FileUtil.stripPath(name);
        }
        log.trace("Client retrieveFile: {}", remoteName);
        if (endpoint.getConfiguration().isStreamDownload()) {
            InputStream is = client.retrieveFileStream(remoteName);
            target.setBody(is);
            exchange.getIn().setHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM, is);
            result = true;
        } else {
            // read the entire file into memory in the byte array
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            result = client.retrieveFile(remoteName, bos);
            // close the stream after done
            IOHelper.close(bos);
            target.setBody(bos.toByteArray());
        }
        // store client reply information after the operation
        exchange.getIn().setHeader(FtpConstants.FTP_REPLY_CODE, client.getReplyCode());
        exchange.getIn().setHeader(FtpConstants.FTP_REPLY_STRING, client.getReplyString());
        // change back to current directory
        if (endpoint.getConfiguration().isStepwise()) {
            changeCurrentDirectory(currentDir);
        }
    } catch (IOException e) {
        throw new GenericFileOperationFailedException(client.getReplyCode(), client.getReplyString(), e.getMessage(), e);
    }
    return result;
}