Examples with FTPFile com.enterprisedt.net.ftp.FTPFile used on opensource projects

Example 1 with FTPFile

use of com.enterprisedt.net.ftp.FTPFile in project pentaho-kettle by pentaho.

the class MVSFileParser method parse.

/**
 * This parses an individual line from the directory listing.
 */
@Override
public FTPFile parse(String raw) throws ParseException {
    String[] aLine = splitMVSLine(raw);
    FTPFile rtn = null;
    if (this.partitionedDataset) {
        // where the real work is done.
        rtn = parsePDSLine(aLine, raw);
    } else {
        // Folder List
        rtn = parseFolder(aLine, raw);
    }
    return rtn;
}

Example 2 with FTPFile

use of com.enterprisedt.net.ftp.FTPFile in project pentaho-kettle by pentaho.

the class MVSFileParser method parsePDSLine.

/**
 ********************** Worker Methods ************************
 */
/**
 * Parses a Partitioned Dataset Entry, and returns an FTPFile object.
 *
 * @param aLine
 *          Split line
 * @param raw
 *          Unparsed raw string
 * @return FTPFile unless it's the header row.
 * @throws ParseException
 */
protected FTPFile parsePDSLine(String[] aLine, String raw) throws ParseException {
    FTPFile rtn = null;
    if (aLine[0].equals(HEADER_NAME)) {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.Header"));
        }
        return null;
    }
    rtn = new FTPFile(raw);
    rtn.setName(aLine[0]);
    if (dateTimeFormat == null) {
        dateTimeFormat = new SimpleDateFormat(dateFormatString + " HH:mm");
    }
    rtn.setCreated(dateFormat.parse(aLine[2]));
    String modDateTime = aLine[3] + ' ' + aLine[4];
    rtn.setLastModified(dateTimeFormat.parse(modDateTime));
    rtn.setDir(false);
    return rtn;
}

Example 3 with FTPFile

use of com.enterprisedt.net.ftp.FTPFile in project pentaho-kettle by pentaho.

the class MockedFTPClient method dirDetails.

@Override
public FTPFile[] dirDetails(String arg0) throws IOException, FTPException, ParseException {
    FTPFile[] files = new FTPFile[10];
    for (int i = 0; i < files.length - 1; i++) {
        files[i] = new FTPFile("file_" + i, "file_" + i, 100, false, new Date());
    }
    files[files.length - 1] = new FTPFile("robots.txt", "robots.txt", 100, false, new Date());
    return files;
}

Example 4 with FTPFile

use of com.enterprisedt.net.ftp.FTPFile in project pentaho-kettle by pentaho.

the class JobEntryFTP method execute.

public Result execute(Result previousResult, int nr) {
    log.logBasic(BaseMessages.getString(PKG, "JobEntryFTP.Started", serverName));
    Result result = previousResult;
    result.setNrErrors(1);
    result.setResult(false);
    NrErrors = 0;
    NrfilesRetrieved = 0;
    successConditionBroken = false;
    boolean exitjobentry = false;
    limitFiles = Const.toInt(environmentSubstitute(getLimit()), 10);
    // Here let's put some controls before stating the job
    if (movefiles) {
        if (Utils.isEmpty(movetodirectory)) {
            logError(BaseMessages.getString(PKG, "JobEntryFTP.MoveToFolderEmpty"));
            return result;
        }
    }
    if (isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.Start"));
    }
    FTPClient ftpclient = null;
    String realMoveToFolder = null;
    try {
        // Create ftp client to host:port ...
        ftpclient = initFTPClient();
        String realServername = environmentSubstitute(serverName);
        String realServerPort = environmentSubstitute(port);
        ftpclient.setRemoteAddr(getInetAddress(realServername));
        if (!Utils.isEmpty(realServerPort)) {
            ftpclient.setRemotePort(Const.toInt(realServerPort, 21));
        }
        if (!Utils.isEmpty(proxyHost)) {
            String realProxy_host = environmentSubstitute(proxyHost);
            ftpclient.setRemoteAddr(InetAddress.getByName(realProxy_host));
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.OpenedProxyConnectionOn", realProxy_host));
            }
            // FIXME: Proper default port for proxy
            int port = Const.toInt(environmentSubstitute(proxyPort), 21);
            if (port != 0) {
                ftpclient.setRemotePort(port);
            }
        } else {
            ftpclient.setRemoteAddr(getInetAddress(realServername));
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.OpenedConnectionTo", realServername));
            }
        }
        // set activeConnection connectmode ...
        if (activeConnection) {
            ftpclient.setConnectMode(FTPConnectMode.ACTIVE);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetActive"));
            }
        } else {
            ftpclient.setConnectMode(FTPConnectMode.PASV);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetPassive"));
            }
        }
        // Set the timeout
        ftpclient.setTimeout(timeout);
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetTimeout", String.valueOf(timeout)));
        }
        ftpclient.setControlEncoding(controlEncoding);
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetEncoding", controlEncoding));
        }
        // If socks proxy server was provided
        if (!Utils.isEmpty(socksProxyHost)) {
            if (!Utils.isEmpty(socksProxyPort)) {
                FTPClient.initSOCKS(environmentSubstitute(socksProxyPort), environmentSubstitute(socksProxyHost));
            } else {
                throw new FTPException(BaseMessages.getString(PKG, "JobEntryFTP.SocksProxy.PortMissingException", environmentSubstitute(socksProxyHost), getName()));
            }
            // then if we have authentication information
            if (!Utils.isEmpty(socksProxyUsername) && !Utils.isEmpty(socksProxyPassword)) {
                FTPClient.initSOCKSAuthentication(environmentSubstitute(socksProxyUsername), Utils.resolvePassword(this, socksProxyPassword));
            } else if (!Utils.isEmpty(socksProxyUsername) && Utils.isEmpty(socksProxyPassword) || Utils.isEmpty(socksProxyUsername) && !Utils.isEmpty(socksProxyPassword)) {
                // we have a username without a password or vica versa
                throw new FTPException(BaseMessages.getString(PKG, "JobEntryFTP.SocksProxy.IncompleteCredentials", environmentSubstitute(socksProxyHost), getName()));
            }
        }
        // login to ftp host ...
        ftpclient.connect();
        String realUsername = environmentSubstitute(userName) + (!Utils.isEmpty(proxyHost) ? "@" + realServername : "") + (!Utils.isEmpty(proxyUsername) ? " " + environmentSubstitute(proxyUsername) : "");
        String realPassword = Utils.resolvePassword(this, password) + (!Utils.isEmpty(proxyPassword) ? " " + Utils.resolvePassword(this, proxyPassword) : "");
        ftpclient.login(realUsername, realPassword);
        // Remove password from logging, you don't know where it ends up.
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.LoggedIn", realUsername));
        }
        // Fix for PDI-2534 - add auxilliary FTP File List parsers to the ftpclient object.
        this.hookInOtherParsers(ftpclient);
        // move to spool dir ...
        if (!Utils.isEmpty(ftpDirectory)) {
            String realFtpDirectory = environmentSubstitute(ftpDirectory);
            realFtpDirectory = normalizePath(realFtpDirectory);
            ftpclient.chdir(realFtpDirectory);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.ChangedDir", realFtpDirectory));
            }
        }
        // Create move to folder if necessary
        if (movefiles && !Utils.isEmpty(movetodirectory)) {
            realMoveToFolder = environmentSubstitute(movetodirectory);
            realMoveToFolder = normalizePath(realMoveToFolder);
            // Folder exists?
            boolean folderExist = true;
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.CheckMoveToFolder", realMoveToFolder));
            }
            String originalLocation = ftpclient.pwd();
            try {
                // does not work for folders, see PDI-2567: folderExist=ftpclient.exists(realMoveToFolder);
                // try switching to the 'move to' folder.
                ftpclient.chdir(realMoveToFolder);
                // Switch back to the previous location.
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.CheckMoveToFolderSwitchBack", originalLocation));
                }
                ftpclient.chdir(originalLocation);
            } catch (Exception e) {
                folderExist = false;
            // Assume folder does not exist !!
            }
            if (!folderExist) {
                if (createmovefolder) {
                    ftpclient.mkdir(realMoveToFolder);
                    if (isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.MoveToFolderCreated", realMoveToFolder));
                    }
                } else {
                    logError(BaseMessages.getString(PKG, "JobEntryFTP.MoveToFolderNotExist"));
                    exitjobentry = true;
                    NrErrors++;
                }
            }
        }
        if (!exitjobentry) {
            // Get all the files in the current directory...
            FTPFile[] ftpFiles = ftpclient.dirDetails(null);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.FoundNFiles", String.valueOf(ftpFiles.length)));
            }
            // set transfertype ...
            if (binaryMode) {
                ftpclient.setType(FTPTransferType.BINARY);
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetBinary"));
                }
            } else {
                ftpclient.setType(FTPTransferType.ASCII);
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "JobEntryFTP.SetAscii"));
                }
            }
            if (ftpFiles.length == 1) {
                String translatedWildcard = environmentSubstitute(wildcard);
                if (!Utils.isEmpty(translatedWildcard)) {
                    if (ftpFiles[0].getName().startsWith(translatedWildcard)) {
                        throw new FTPException(ftpFiles[0].getName());
                    }
                }
            }
            Pattern pattern = null;
            if (!Utils.isEmpty(wildcard)) {
                String realWildcard = environmentSubstitute(wildcard);
                pattern = Pattern.compile(realWildcard);
            }
            if (!getSuccessCondition().equals(SUCCESS_IF_NO_ERRORS)) {
                limitFiles = Const.toInt(environmentSubstitute(getLimit()), 10);
            }
            // Get the files in the list...
            for (FTPFile ftpFile : ftpFiles) {
                if (parentJob.isStopped()) {
                    exitjobentry = true;
                    throw new Exception(BaseMessages.getString(PKG, "JobEntryFTP.JobStopped"));
                }
                if (successConditionBroken) {
                    throw new Exception(BaseMessages.getString(PKG, "JobEntryFTP.SuccesConditionBroken", "" + NrErrors));
                }
                boolean getIt = true;
                String filename = ftpFile.getName();
                if (isDebug()) {
                    logDebug(BaseMessages.getString(PKG, "JobEntryFTP.AnalysingFile", filename));
                }
                // We get only files
                if (ftpFile.isDir()) {
                    // not a file..so let's skip it!
                    getIt = false;
                    if (isDebug()) {
                        logDebug(BaseMessages.getString(PKG, "JobEntryFTP.SkippingNotAFile", filename));
                    }
                }
                if (getIt) {
                    try {
                        // See if the file matches the regular expression!
                        if (pattern != null) {
                            Matcher matcher = pattern.matcher(filename);
                            getIt = matcher.matches();
                        }
                        if (getIt) {
                            downloadFile(ftpclient, filename, realMoveToFolder, parentJob, result);
                        }
                    } catch (Exception e) {
                        // Update errors number
                        updateErrors();
                        logError(BaseMessages.getString(PKG, "JobFTP.UnexpectedError", e.toString()));
                    }
                }
            }
        // end for
        }
    } catch (Exception e) {
        if (!successConditionBroken && !exitjobentry) {
            updateErrors();
        }
        logError(BaseMessages.getString(PKG, "JobEntryFTP.ErrorGetting", e.getMessage()));
    } finally {
        if (ftpclient != null) {
            try {
                ftpclient.quit();
            } catch (Exception e) {
                logError(BaseMessages.getString(PKG, "JobEntryFTP.ErrorQuitting", e.getMessage()));
            }
        }
        FTPClient.clearSOCKS();
    }
    result.setNrErrors(NrErrors);
    result.setNrFilesRetrieved(NrfilesRetrieved);
    if (getSuccessStatus()) {
        result.setResult(true);
    }
    if (exitjobentry) {
        result.setResult(false);
    }
    displayResults();
    return result;
}

Example 5 with FTPFile

use of com.enterprisedt.net.ftp.FTPFile in project pentaho-kettle by pentaho.

the class MVSFileParser method parseFolder.

/**
 * Parses a line from a folder listing.
 *
 * Note: Returns NULL if it's the header line, if it is ARCIVE or Migrated, if the record format doesn't start with
 * 'F' or 'V', and if the dsorg doesn't start with 'P'.
 *
 * @param aLine
 *          Line split apart
 * @param raw
 *          Raw line from the transport
 * @return FTPFile for the line unless it is expressly exluded
 */
protected FTPFile parseFolder(String[] aLine, String raw) {
    if (aLine[0].equals(HEADER_VOLUME)) {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.Header"));
        }
        return null;
    }
    // Directory format
    if (aLine[0].equals(LINE_TYPE_ARCIVE)) {
        // It's on tape somewhere
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.ARCIVE"));
        }
        return null;
    }
    if (aLine[0].equals(LINE_TYPE_MIGRATED)) {
        // It's been moved.
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.Migrated"));
        }
        return null;
    }
    if (aLine[5].charAt(0) != 'F' && aLine[5].charAt(0) != 'V') {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.recf"));
        }
        return null;
    }
    if (aLine[8].charAt(0) != 'P') {
        // Only handle PO, PS, or PO-E
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Skip.dso"));
        }
        return null;
    }
    // OK, I think I can handle this.
    FTPFile rtn = new FTPFile(raw);
    rtn.setName(aLine[9]);
    // Fake out dates - these are all newly created files / folders
    rtn.setCreated(new Date());
    rtn.setLastModified(new Date());
    if (aLine[8].equals(ENTRY_FILE_TYPE)) {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Found.File", aLine[9]));
        }
        // This is a file...
        rtn.setDir(false);
        long l = -1;
        try {
            l = Long.parseLong(aLine[4]);
        } catch (Exception ignored) {
        // Ignore errors
        }
        rtn.setSize(l);
    } else {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "MVSFileParser.DEBUG.Found.Folder", aLine[9]));
        }
        rtn.setDir(true);
    }
    // 
    return rtn;
}