Examples with SFTPv3Client com.trilead.ssh2.SFTPv3Client used on opensource projects

Example 16 with SFTPv3Client

use of com.trilead.ssh2.SFTPv3Client in project pentaho-kettle by pentaho.

the class JobEntrySSH2GET method copyRecursive.

/**
 * copy a directory from the remote host to the local one recursivly.
 *
 * @param sourceLocation
 *          the source directory on the remote host
 * @param targetLocation
 *          the target directory on the local host
 * @param sftpClient
 *          is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
 * @return the number of files successfully copied
 * @throws Exception
 */
private void copyRecursive(String sourceLocation, String targetLocation, SFTPv3Client sftpClient, Pattern pattern, Job parentJob) throws Exception {
    String sourceFolder = "." + FTPUtils.FILE_SEPARATOR;
    if (sourceLocation != null) {
        sourceFolder = sourceLocation;
    }
    if (this.isDirectory(sftpClient, sourceFolder)) {
        Vector<?> filelist = sftpClient.ls(sourceFolder);
        Iterator<?> iterator = filelist.iterator();
        while (iterator.hasNext()) {
            SFTPv3DirectoryEntry dirEntry = (SFTPv3DirectoryEntry) iterator.next();
            if (dirEntry == null) {
                continue;
            }
            if (dirEntry.filename.equals(".") || dirEntry.filename.equals("..")) {
                continue;
            }
            copyRecursive(sourceFolder + FTPUtils.FILE_SEPARATOR + dirEntry.filename, targetLocation + Const.FILE_SEPARATOR + dirEntry.filename, sftpClient, pattern, parentJob);
        }
    } else if (isFile(sftpClient, sourceFolder)) {
        if (getFileWildcard(sourceFolder, pattern)) {
            copyFile(sourceFolder, targetLocation, sftpClient);
        }
    }
}

Example 17 with SFTPv3Client

use of com.trilead.ssh2.SFTPv3Client in project pentaho-kettle by pentaho.

the class JobEntrySSH2GET method copyFile.

/**
 * @param sourceLocation
 * @param targetLocation
 * @param sftpClient
 * @return
 */
private void copyFile(String sourceLocation, String targetLocation, SFTPv3Client sftpClient) {
    SFTPv3FileHandle sftpFileHandle = null;
    FileOutputStream fos = null;
    File transferFile = null;
    long remoteFileSize = -1;
    boolean filecopied = true;
    try {
        transferFile = new File(targetLocation);
        if ((onlyGettingNewFiles == false) || (onlyGettingNewFiles == true) && !FileExists(transferFile.getAbsolutePath())) {
            new File(transferFile.getParent()).mkdirs();
            remoteFileSize = this.getFileSize(sftpClient, sourceLocation);
            if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.ReceivingFile", sourceLocation, transferFile.getAbsolutePath(), "" + remoteFileSize));
            }
            sftpFileHandle = sftpClient.openFileRO(sourceLocation);
            fos = null;
            long offset = 0;
            fos = new FileOutputStream(transferFile);
            byte[] buffer = new byte[2048];
            while (true) {
                int len = sftpClient.read(sftpFileHandle, offset, buffer, 0, buffer.length);
                if (len <= 0) {
                    break;
                }
                fos.write(buffer, 0, len);
                offset += len;
            }
            fos.flush();
            fos.close();
            fos = null;
            nbfilestoget++;
            if (remoteFileSize > 0 && remoteFileSize != transferFile.length()) {
                filecopied = false;
                nbrerror++;
                logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.Error.RemoteFileLocalDifferent", "" + remoteFileSize, transferFile.length() + "", "" + offset));
            } else {
                nbgot++;
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RemoteFileLocalCopied", sourceLocation, transferFile + ""));
                }
            }
        }
        // Let's now delete or move file if needed...
        if (filecopied && !afterFtpPut.equals("do_nothing")) {
            deleteOrMoveFiles(sftpClient, sourceLocation, environmentSubstitute(destinationfolder));
        }
    } catch (Exception e) {
        nbrerror++;
        logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.Error.WritingFile", transferFile.getAbsolutePath(), e.getMessage()));
    } finally {
        try {
            if (sftpFileHandle != null) {
                sftpClient.closeFile(sftpFileHandle);
                sftpFileHandle = null;
            }
            if (fos != null) {
                try {
                    fos.close();
                    fos = null;
                } catch (Exception ex) {
                // Ignore errors
                }
            }
        } catch (Exception e) {
        // Ignore errors
        }
    }
}

Example 18 with SFTPv3Client

use of com.trilead.ssh2.SFTPv3Client in project pentaho-kettle by pentaho.

the class JobEntrySSH2PUT method putFile.

private boolean putFile(FileObject localFile, String remotefilename, SFTPv3Client sftpClient) {
    long filesize = -1;
    InputStream in = null;
    BufferedInputStream inBuf = null;
    SFTPv3FileHandle sftpFileHandle = null;
    boolean retval = false;
    try {
        // Put file in the folder
        sftpFileHandle = sftpClient.createFileTruncate(remotefilename);
        // Associate a file input stream for the current local file
        in = KettleVFS.getInputStream(localFile);
        inBuf = new BufferedInputStream(in);
        byte[] buf = new byte[2048];
        long offset = 0;
        long length = localFile.getContent().getSize();
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.SendingFile", localFile.toString(), "" + length, remotefilename));
        }
        // Write to remote file
        while (true) {
            int len = in.read(buf, 0, buf.length);
            if (len <= 0) {
                break;
            }
            sftpClient.write(sftpFileHandle, offset, buf, 0, len);
            offset += len;
        }
        // Get File size
        filesize = getFileSize(sftpClient, remotefilename);
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.FileOnRemoteHost", remotefilename, "" + filesize));
        }
        retval = true;
    } catch (Exception e) {
        // We failed to put files
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ErrorCopyingFile", localFile.toString()) + ":" + e.getMessage());
    } finally {
        if (in != null) {
            try {
                in.close();
                in = null;
            } catch (Exception ex) {
            // Ignore errors
            }
        }
        if (inBuf != null) {
            try {
                inBuf.close();
                inBuf = null;
            } catch (Exception ex) {
            // Ignore errors
            }
        }
        if (sftpFileHandle != null) {
            try {
                sftpClient.closeFile(sftpFileHandle);
                sftpFileHandle = null;
            } catch (Exception ex) {
            // Ignore errors
            }
        }
    }
    return retval;
}

Example 19 with SFTPv3Client

use of com.trilead.ssh2.SFTPv3Client in project pentaho-kettle by pentaho.

the class JobEntrySSH2PUT method execute.

@Override
public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);
    try {
        // Get real variable value
        String realServerName = environmentSubstitute(serverName);
        int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
        String realUserName = environmentSubstitute(userName);
        String realServerPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
        // Proxy Host
        String realProxyHost = environmentSubstitute(httpproxyhost);
        int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
        String realproxyUserName = environmentSubstitute(httpproxyusername);
        String realProxyPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
        // Key file
        String realKeyFilename = environmentSubstitute(keyFilename);
        String relKeyFilepass = environmentSubstitute(keyFilePass);
        // Source files
        String realLocalDirectory = environmentSubstitute(localDirectory);
        String realwildcard = environmentSubstitute(wildcard);
        // Remote destination
        String realftpDirectory = environmentSubstitute(ftpDirectory);
        // Destination folder (Move to)
        String realDestinationFolder = environmentSubstitute(destinationfolder);
        try {
            // Remote source
            realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
            // Destination folder (Move to)
            realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
        } catch (Exception e) {
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
            result.setNrErrors(1);
            return result;
        }
        // Check for mandatory fields
        boolean mandatoryok = true;
        if (Utils.isEmpty(realServerName)) {
            mandatoryok = false;
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
        }
        if (usehttpproxy) {
            if (Utils.isEmpty(realProxyHost)) {
                mandatoryok = false;
                logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
            }
        }
        if (publicpublickey) {
            if (Utils.isEmpty(realKeyFilename)) {
                mandatoryok = false;
                logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
            } else {
                // Let's check if folder exists...
                if (!KettleVFS.fileExists(realKeyFilename, this)) {
                    mandatoryok = false;
                    logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
                }
            }
        }
        if (Utils.isEmpty(realLocalDirectory)) {
            mandatoryok = false;
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
        }
        if (afterFtpPut.equals("move_file")) {
            if (Utils.isEmpty(realDestinationFolder)) {
                mandatoryok = false;
                logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
            } else {
                FileObject folder = null;
                try {
                    folder = KettleVFS.getFileObject(realDestinationFolder, this);
                    // Let's check if folder exists...
                    if (!folder.exists()) {
                        // Do we need to create it?
                        if (createDestinationFolder) {
                            folder.createFolder();
                        } else {
                            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
                        }
                    }
                } catch (Exception e) {
                    throw new KettleException(e);
                } finally {
                    if (folder != null) {
                        try {
                            folder.close();
                            folder = null;
                        } catch (Exception e) {
                        /* Ignore */
                        }
                    }
                }
            }
        }
        if (mandatoryok) {
            Connection conn = null;
            SFTPv3Client client = null;
            boolean good = true;
            int nbfilestoput = 0;
            int nbput = 0;
            int nbrerror = 0;
            try {
                // Create a connection instance
                conn = getConnection(realServerName, realServerPort, realProxyHost, realProxyPort, realproxyUserName, realProxyPassword);
                if (timeout > 0) {
                    // Cache Host Key
                    if (cachehostkey) {
                        conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
                    } else {
                        conn.connect(null, 0, timeout * 1000);
                    }
                } else {
                    // Cache Host Key
                    if (cachehostkey) {
                        conn.connect(new SimpleVerifier(database));
                    } else {
                        conn.connect();
                    }
                }
                // Authenticate
                boolean isAuthenticated = false;
                if (publicpublickey) {
                    String keyContent = KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
                    isAuthenticated = conn.authenticateWithPublicKey(realUserName, keyContent.toCharArray(), relKeyFilepass);
                } else {
                    isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
                }
                // LET'S CHECK AUTHENTICATION ...
                if (isAuthenticated == false) {
                    logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
                } else {
                    if (log.isBasic()) {
                        logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
                    }
                    client = new SFTPv3Client(conn);
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
                    }
                    // Check if remote directory exists
                    if (!Utils.isEmpty(realftpDirectory)) {
                        if (!sshDirectoryExists(client, realftpDirectory)) {
                            good = false;
                            if (createRemoteFolder) {
                                good = CreateRemoteFolder(client, realftpDirectory);
                                if (good) {
                                    logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
                                }
                            } else {
                                logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
                            }
                        } else if (log.isDetailed()) {
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
                        }
                    }
                    if (good) {
                        // Get files list from local folder (source)
                        List<FileObject> myFileList = getFiles(realLocalDirectory);
                        // Prepare Pattern for wildcard
                        Pattern pattern = null;
                        if (!Utils.isEmpty(realwildcard)) {
                            pattern = Pattern.compile(realwildcard);
                        }
                        // Get the files in the list
                        for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
                            FileObject myFile = myFileList.get(i);
                            String localFilename = myFile.toString();
                            String remoteFilename = myFile.getName().getBaseName();
                            boolean getIt = true;
                            // First see if the file matches the regular expression!
                            if (pattern != null) {
                                Matcher matcher = pattern.matcher(remoteFilename);
                                getIt = matcher.matches();
                            }
                            // do we have a target directory?
                            if (!Utils.isEmpty(realftpDirectory)) {
                                remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
                            }
                            if (onlyGettingNewFiles) {
                                // We get only new files
                                // ie not exist on the remote server
                                getIt = !sshFileExists(client, remoteFilename);
                            }
                            if (getIt) {
                                nbfilestoput++;
                                boolean putok = putFile(myFile, remoteFilename, client);
                                if (!putok) {
                                    nbrerror++;
                                    logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
                                } else {
                                    nbput++;
                                }
                                if (putok && !afterFtpPut.equals("do_nothing")) {
                                    deleteOrMoveFiles(myFile, realDestinationFolder);
                                }
                            }
                        }
                        /**
                         ****************************** RESULT *******************
                         */
                        if (log.isDetailed()) {
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
                            logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
                        }
                        if (nbrerror == 0) {
                            result.setResult(true);
                        /**
                         ****************************** RESULT *******************
                         */
                        }
                    }
                }
            } catch (Exception e) {
                result.setNrErrors(nbrerror);
                logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
            } finally {
                if (conn != null) {
                    conn.close();
                }
                if (client != null) {
                    client.close();
                }
            }
        }
    } catch (Exception e) {
        result.setResult(false);
        result.setNrErrors(1L);
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
    }
    return result;
}