use of org.apache.commons.vfs2.VFS in project pentaho-kettle by pentaho.
the class JobEntrySFTP method execute.
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
RowMetaAndData resultRow = null;
result.setResult(false);
long filesRetrieved = 0;
// Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
if (parentJobMeta.getNamedClusterEmbedManager() != null) {
parentJobMeta.getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, parentJobMeta.getEmbeddedMetastoreProviderKey());
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.StartJobEntry"));
}
HashSet<String> list_previous_filenames = new HashSet<String>();
if (copyprevious) {
if (rows.size() == 0) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.ArgsFromPreviousNothing"));
}
result.setResult(true);
return result;
}
try {
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size(); iteration++) {
resultRow = rows.get(iteration);
// Get file names
String file_previous = resultRow.getString(0, null);
if (!Utils.isEmpty(file_previous)) {
list_previous_filenames.add(file_previous);
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "JobSFTP.Log.FilenameFromResult", file_previous));
}
}
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSFTP.Error.ArgFromPrevious"));
result.setNrErrors(1);
return result;
}
}
SFTPClient sftpclient = null;
// String substitution..
String realServerName = environmentSubstitute(serverName);
String realServerPort = environmentSubstitute(serverPort);
String realUsername = environmentSubstitute(userName);
String realPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
String realSftpDirString = environmentSubstitute(sftpDirectory);
String realWildcard = environmentSubstitute(wildcard);
String realTargetDirectory = environmentSubstitute(targetDirectory);
String realKeyFilename = null;
String realPassPhrase = null;
FileObject TargetFolder = null;
try {
// Let's perform some checks before starting
if (isUseKeyFile()) {
// We must have here a private keyfilename
realKeyFilename = environmentSubstitute(getKeyFilename());
if (Utils.isEmpty(realKeyFilename)) {
// Error..Missing keyfile
logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileMissing"));
result.setNrErrors(1);
return result;
}
if (!KettleVFS.fileExists(realKeyFilename)) {
// Error.. can not reach keyfile
logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileNotFound", realKeyFilename));
result.setNrErrors(1);
return result;
}
realPassPhrase = environmentSubstitute(getKeyPassPhrase());
}
if (!Utils.isEmpty(realTargetDirectory)) {
TargetFolder = KettleVFS.getFileObject(realTargetDirectory, this);
boolean TargetFolderExists = TargetFolder.exists();
if (TargetFolderExists) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.TargetFolderExists", realTargetDirectory));
}
} else {
if (!createtargetfolder) {
// Error..Target folder can not be found !
logError(BaseMessages.getString(PKG, "JobSFTP.Error.TargetFolderNotExists", realTargetDirectory));
result.setNrErrors(1);
return result;
} else {
// create target folder
TargetFolder.createFolder();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.TargetFolderCreated", realTargetDirectory));
}
}
}
}
if (TargetFolder != null) {
TargetFolder.close();
TargetFolder = null;
}
// Create sftp client to host ...
sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, DEFAULT_PORT), realUsername, realKeyFilename, realPassPhrase);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.OpenedConnection", realServerName, realServerPort, realUsername));
}
// Set compression
sftpclient.setCompression(getCompression());
// Set proxy?
String realProxyHost = environmentSubstitute(getProxyHost());
if (!Utils.isEmpty(realProxyHost)) {
// Set proxy
String password = getRealPassword(getProxyPassword());
sftpclient.setProxy(realProxyHost, environmentSubstitute(getProxyPort()), environmentSubstitute(getProxyUsername()), password, getProxyType());
}
// login to ftp host ...
sftpclient.login(realPassword);
// move to spool dir ...
if (!Utils.isEmpty(realSftpDirString)) {
try {
sftpclient.chdir(realSftpDirString);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSFTP.Error.CanNotFindRemoteFolder", realSftpDirString));
throw new Exception(e);
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.ChangedDirectory", realSftpDirString));
}
}
Pattern pattern = null;
// Get all the files in the current directory...
String[] filelist = sftpclient.dir();
if (filelist == null) {
// Nothing was found !!! exit
result.setResult(true);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.Found", "" + 0));
}
return result;
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.Found", "" + filelist.length));
}
if (!copyprevious) {
if (!Utils.isEmpty(realWildcard)) {
pattern = Pattern.compile(realWildcard);
}
}
// Get the files in the list...
for (int i = 0; i < filelist.length && !parentJob.isStopped(); i++) {
boolean getIt = true;
if (copyprevious) {
// filenames list is send by previous job entry
// download if the current file is in this list
getIt = list_previous_filenames.contains(filelist[i]);
} else {
// but before see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(filelist[i]);
getIt = matcher.matches();
}
}
if (getIt) {
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "JobSFTP.Log.GettingFiles", filelist[i], realTargetDirectory));
}
FileObject targetFile = KettleVFS.getFileObject(realTargetDirectory + Const.FILE_SEPARATOR + filelist[i], this);
sftpclient.get(targetFile, filelist[i]);
filesRetrieved++;
if (isaddresult) {
// Add to the result files...
ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, targetFile, parentJob.getJobname(), toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.FilenameAddedToResultFilenames", filelist[i]));
}
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.TransferedFile", filelist[i]));
}
// Delete the file if this is needed!
if (remove) {
sftpclient.delete(filelist[i]);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTP.Log.DeletedFile", filelist[i]));
}
}
}
}
result.setResult(true);
result.setNrFilesRetrieved(filesRetrieved);
} catch (Exception e) {
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobSFTP.Error.GettingFiles", e.getMessage()));
logError(Const.getStackTracker(e));
} finally {
// close connection, if possible
try {
if (sftpclient != null) {
sftpclient.disconnect();
}
} catch (Exception e) {
// just ignore this, makes no big difference
}
try {
if (TargetFolder != null) {
TargetFolder.close();
TargetFolder = null;
}
if (list_previous_filenames != null) {
list_previous_filenames = null;
}
} catch (Exception e) {
// Ignore errors
}
}
return result;
}
use of org.apache.commons.vfs2.VFS in project pentaho-kettle by pentaho.
the class JobEntrySFTPPUT method execute.
public Result execute(Result previousResult, int nr) throws KettleException {
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
result.setResult(false);
// Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
if (parentJobMeta.getNamedClusterEmbedManager() != null) {
parentJobMeta.getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, parentJobMeta.getEmbeddedMetastoreProviderKey());
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.StartJobEntry"));
}
ArrayList<FileObject> myFileList = new ArrayList<FileObject>();
if (copyprevious) {
if (rows.size() == 0) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothing"));
}
result.setResult(true);
return result;
}
try {
RowMetaAndData resultRow = null;
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size(); iteration++) {
resultRow = rows.get(iteration);
// Get file names
String file_previous = resultRow.getString(0, null);
if (!Utils.isEmpty(file_previous)) {
FileObject file = KettleVFS.getFileObject(file_previous, this);
if (!file.exists()) {
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file_previous));
} else {
myFileList.add(file);
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file_previous));
}
}
}
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious"));
result.setNrErrors(1);
// free resource
myFileList = null;
return result;
}
}
if (copypreviousfiles) {
List<ResultFile> resultFiles = result.getResultFilesList();
if (resultFiles == null || resultFiles.size() == 0) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothingFiles"));
}
result.setResult(true);
return result;
}
try {
for (Iterator<ResultFile> it = resultFiles.iterator(); it.hasNext() && !parentJob.isStopped(); ) {
ResultFile resultFile = it.next();
FileObject file = resultFile.getFile();
if (file != null) {
if (!file.exists()) {
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file.toString()));
} else {
myFileList.add(file);
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file.toString()));
}
}
}
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious"));
result.setNrErrors(1);
// free resource
myFileList = null;
return result;
}
}
SFTPClient sftpclient = null;
// String substitution..
String realServerName = environmentSubstitute(serverName);
String realServerPort = environmentSubstitute(serverPort);
String realUsername = environmentSubstitute(userName);
String realPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
String realSftpDirString = environmentSubstitute(sftpDirectory);
String realWildcard = environmentSubstitute(wildcard);
String realLocalDirectory = environmentSubstitute(localDirectory);
String realKeyFilename = null;
String realPassPhrase = null;
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(getDestinationFolder());
try {
if (getAfterFTPS() == AFTER_FTPSPUT_MOVE) {
if (Utils.isEmpty(realDestinationFolder)) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
result.setNrErrors(1);
return result;
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
result.setNrErrors(1);
return result;
}
}
realDestinationFolder = KettleVFS.getFilename(folder);
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (isUseKeyFile()) {
// We must have here a private keyfilename
realKeyFilename = environmentSubstitute(getKeyFilename());
if (Utils.isEmpty(realKeyFilename)) {
// Error..Missing keyfile
logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileMissing"));
result.setNrErrors(1);
return result;
}
if (!KettleVFS.fileExists(realKeyFilename)) {
// Error.. can not reach keyfile
logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileNotFound"));
result.setNrErrors(1);
return result;
}
realPassPhrase = environmentSubstitute(getKeyPassPhrase());
}
// Create sftp client to host ...
sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername, realKeyFilename, realPassPhrase);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername));
}
// Set compression
sftpclient.setCompression(getCompression());
// Set proxy?
String realProxyHost = environmentSubstitute(getProxyHost());
if (!Utils.isEmpty(realProxyHost)) {
// Set proxy
sftpclient.setProxy(realProxyHost, environmentSubstitute(getProxyPort()), environmentSubstitute(getProxyUsername()), environmentSubstitute(getProxyPassword()), getProxyType());
}
// login to ftp host ...
sftpclient.login(realPassword);
// move to spool dir ...
if (!Utils.isEmpty(realSftpDirString)) {
boolean existfolder = sftpclient.folderExists(realSftpDirString);
if (!existfolder) {
if (!isCreateRemoteFolder()) {
throw new KettleException(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString));
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString));
}
// Let's create folder
sftpclient.createFolder(realSftpDirString);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.RemoteFolderCreated", realSftpDirString));
}
}
sftpclient.chdir(realSftpDirString);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.ChangedDirectory", realSftpDirString));
}
}
if (!copyprevious && !copypreviousfiles) {
// Get all the files in the local directory...
myFileList = new ArrayList<FileObject>();
FileObject localFiles = KettleVFS.getFileObject(realLocalDirectory, this);
FileObject[] children = localFiles.getChildren();
if (children != null) {
for (int i = 0; i < children.length; i++) {
// Get filename of file or directory
if (children[i].getType().equals(FileType.FILE)) {
// myFileList.add(children[i].getAbsolutePath());
myFileList.add(children[i]);
}
}
// end for
}
}
if (myFileList == null || myFileList.size() == 0) {
if (isSuccessWhenNoFile()) {
// Just warn user
if (isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend"));
}
} else {
// Fail
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend"));
result.setNrErrors(1);
return result;
}
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.RowsFromPreviousResult", myFileList.size()));
}
Pattern pattern = null;
if (!copyprevious && !copypreviousfiles) {
if (!Utils.isEmpty(realWildcard)) {
pattern = Pattern.compile(realWildcard);
}
}
// Get the files in the list and execute sftp.put() for each file
Iterator<FileObject> it = myFileList.iterator();
while (it.hasNext() && !parentJob.isStopped()) {
FileObject myFile = it.next();
try {
String localFilename = myFile.toString();
String destinationFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(destinationFilename);
getIt = matcher.matches();
}
if (getIt) {
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.PuttingFile", localFilename, realSftpDirString));
}
sftpclient.put(myFile, destinationFilename);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.TransferedFile", localFilename));
}
// what's next ...
switch(getAfterFTPS()) {
case AFTER_FTPSPUT_DELETE:
myFile.delete();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.DeletedFile", localFilename));
}
break;
case AFTER_FTPSPUT_MOVE:
FileObject destination = null;
try {
destination = KettleVFS.getFileObject(realDestinationFolder + Const.FILE_SEPARATOR + myFile.getName().getBaseName(), this);
myFile.moveTo(destination);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FileMoved", myFile, destination));
}
} finally {
if (destination != null) {
destination.close();
}
}
break;
default:
if (addFilenameResut) {
// Add to the result files...
ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentJob.getJobname(), toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameAddedToResultFilenames", localFilename));
}
}
break;
}
}
} finally {
if (myFile != null) {
myFile.close();
}
}
}
// end for
result.setResult(true);
// JKU: no idea if this is needed...!
// result.setNrFilesRetrieved(filesRetrieved);
} catch (Exception e) {
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobSFTPPUT.Exception", e.getMessage()));
logError(Const.getStackTracker(e));
} finally {
// close connection, if possible
try {
if (sftpclient != null) {
sftpclient.disconnect();
}
} catch (Exception e) {
// just ignore this, makes no big difference
}
// end catch
myFileList = null;
}
return result;
}
use of org.apache.commons.vfs2.VFS in project pentaho-kettle by pentaho.
the class SwingSvgImageUtil method loadFromBasedVFS.
/**
* Internal image loading from Kettle's user.dir VFS.
*/
private static SwingUniversalImage loadFromBasedVFS(String location) {
try {
FileObject imageFileObject = KettleVFS.getInstance().getFileSystemManager().resolveFile(base, location);
InputStream s = KettleVFS.getInputStream(imageFileObject);
if (s == null) {
return null;
}
try {
return loadImage(s, location);
} finally {
IOUtils.closeQuietly(s);
}
} catch (FileSystemException ex) {
return null;
}
}
use of org.apache.commons.vfs2.VFS in project pentaho-kettle by pentaho.
the class JobEntryCreateFile method execute.
public Result execute(Result previousResult, int nr) throws KettleException {
Result result = previousResult;
result.setResult(false);
if (filename != null) {
// Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
if (parentJobMeta.getNamedClusterEmbedManager() != null) {
parentJobMeta.getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, parentJobMeta.getEmbeddedMetastoreProviderKey());
}
String realFilename = getRealFilename();
FileObject fileObject = null;
try {
fileObject = KettleVFS.getFileObject(realFilename, this);
if (fileObject.exists()) {
if (isFailIfFileExists()) {
// File exists and fail flag is on.
result.setResult(false);
logError("File [" + realFilename + "] exists, failing.");
} else {
// File already exists, no reason to try to create it
result.setResult(true);
logBasic("File [" + realFilename + "] already exists, not recreating.");
}
// add filename to result filenames if needed
if (isAddFilenameToResult()) {
addFilenameToResult(realFilename, result, parentJob);
}
} else {
// No file yet, create an empty file.
fileObject.createFile();
logBasic("File [" + realFilename + "] created!");
// add filename to result filenames if needed
if (isAddFilenameToResult()) {
addFilenameToResult(realFilename, result, parentJob);
}
result.setResult(true);
}
} catch (IOException e) {
logError("Could not create file [" + realFilename + "], exception: " + e.getMessage());
result.setResult(false);
result.setNrErrors(1);
} finally {
if (fileObject != null) {
try {
fileObject.close();
fileObject = null;
} catch (IOException ex) {
// Ignore
}
}
}
} else {
logError("No filename is defined.");
}
return result;
}
use of org.apache.commons.vfs2.VFS in project pentaho-kettle by pentaho.
the class JobEntryCreateFolder method execute.
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
if (foldername != null) {
// Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
if (parentJobMeta.getNamedClusterEmbedManager() != null) {
parentJobMeta.getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, parentJobMeta.getEmbeddedMetastoreProviderKey());
}
String realFoldername = getRealFoldername();
FileObject folderObject = null;
try {
folderObject = KettleVFS.getFileObject(realFoldername, this);
if (folderObject.exists()) {
boolean isFolder = false;
// Check if it's a folder
if (folderObject.getType() == FileType.FOLDER) {
isFolder = true;
}
if (isFailOfFolderExists()) {
// Folder exists and fail flag is on.
result.setResult(false);
if (isFolder) {
logError("Folder [" + realFoldername + "] exists, failing.");
} else {
logError("File [" + realFoldername + "] exists, failing.");
}
} else {
// Folder already exists, no reason to try to create it
result.setResult(true);
if (log.isDetailed()) {
logDetailed("Folder [" + realFoldername + "] already exists, not recreating.");
}
}
} else {
// No Folder yet, create an empty Folder.
folderObject.createFolder();
if (log.isDetailed()) {
logDetailed("Folder [" + realFoldername + "] created!");
}
result.setResult(true);
}
} catch (Exception e) {
logError("Could not create Folder [" + realFoldername + "]", e);
result.setResult(false);
result.setNrErrors(1);
} finally {
if (folderObject != null) {
try {
folderObject.close();
} catch (IOException ex) {
/* Ignore */
}
}
}
} else {
logError("No Foldername is defined.");
}
return result;
}
Aggregations