use of org.apache.hop.core.exception.HopException in project hop by apache.
the class TransformStatus method getXml.
@JsonIgnore
public String getXml() throws HopException {
try {
StringBuilder xml = new StringBuilder();
xml.append(XmlHandler.openTag(XML_TAG));
xml.append(XmlHandler.addTagValue("transformName", transformName, false));
xml.append(XmlHandler.addTagValue("copy", copy, false));
xml.append(XmlHandler.addTagValue("linesRead", linesRead, false));
xml.append(XmlHandler.addTagValue("linesWritten", linesWritten, false));
xml.append(XmlHandler.addTagValue("linesInput", linesInput, false));
xml.append(XmlHandler.addTagValue("linesOutput", linesOutput, false));
xml.append(XmlHandler.addTagValue("linesUpdated", linesUpdated, false));
xml.append(XmlHandler.addTagValue("linesRejected", linesRejected, false));
xml.append(XmlHandler.addTagValue("errors", errors, false));
xml.append(XmlHandler.addTagValue("input_buffer_size", inputBufferSize, false));
xml.append(XmlHandler.addTagValue("output_buffer_size", outputBufferSize, false));
xml.append(XmlHandler.addTagValue("statusDescription", statusDescription, false));
xml.append(XmlHandler.addTagValue("seconds", seconds, false));
xml.append(XmlHandler.addTagValue("speed", speed, false));
xml.append(XmlHandler.addTagValue("priority", priority, false));
xml.append(XmlHandler.addTagValue("stopped", stopped, false));
xml.append(XmlHandler.addTagValue("paused", paused, false));
if (sampleRowMeta != null) {
xml.append(XmlHandler.openTag("samples"));
xml.append(sampleRowMeta.getMetaXml());
xml.append(Const.CR);
if (sampleRows != null) {
synchronized (sampleRows) {
Iterator<Object[]> iterator = sampleRows.iterator();
while (iterator.hasNext()) {
Object[] sampleRow = iterator.next();
xml.append(sampleRowMeta.getDataXml(sampleRow));
xml.append(Const.CR);
}
}
}
xml.append(XmlHandler.closeTag("samples"));
}
xml.append(XmlHandler.closeTag(XML_TAG));
return xml.toString();
} catch (Exception e) {
throw new HopException("Unable to serialize transform '" + transformName + "' status data to XML", e);
}
}
use of org.apache.hop.core.exception.HopException in project hop by apache.
the class AbstractFileErrorHandler method getWriter.
/**
* returns the OutputWiter if exists. Otherwhise it will create a new one.
*
* @return
* @throws HopException
*/
Writer getWriter(Object source) throws HopException {
try {
Writer outputStreamWriter = writers.get(source);
if (outputStreamWriter != null) {
return outputStreamWriter;
}
FileObject file = getReplayFilename(destinationDirectory, processingFilename, dateString, fileExtension, source);
ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, file, baseTransform.getPipelineMeta().getName(), baseTransform.getTransformName());
baseTransform.addResultFile(resultFile);
try {
if (encoding == null) {
outputStreamWriter = new OutputStreamWriter(HopVfs.getOutputStream(file, false));
} else {
outputStreamWriter = new OutputStreamWriter(HopVfs.getOutputStream(file, false), encoding);
}
} catch (Exception e) {
throw new HopException(BaseMessages.getString(PKG, "AbstractFileErrorHandler.Exception.CouldNotCreateFileErrorHandlerForFile") + file.getName().getURI(), e);
}
writers.put(source, outputStreamWriter);
return outputStreamWriter;
} catch (HopFileException e) {
throw new HopException(BaseMessages.getString(PKG, "AbstractFileErrorHandler.Exception.CouldNotCreateFileErrorHandlerForFile"), e);
}
}
use of org.apache.hop.core.exception.HopException in project hop by apache.
the class ActionSftpPut method execute.
@Override
public Result execute(Result previousResult, int nr) throws HopException {
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
result.setResult(false);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.StartAction"));
}
ArrayList<FileObject> myFileList = new ArrayList<>();
if (copyingPrevious) {
if (rows.size() == 0) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.ArgsFromPreviousNothing"));
}
result.setResult(true);
return result;
}
try {
RowMetaAndData resultRow = null;
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size(); iteration++) {
resultRow = rows.get(iteration);
// Get file names
String filePrevious = resultRow.getString(0, null);
if (!Utils.isEmpty(filePrevious)) {
FileObject file = HopVfs.getFileObject(filePrevious);
if (!file.exists()) {
logError(BaseMessages.getString(PKG, "ActionSftpPut.Log.FilefromPreviousNotFound", filePrevious));
} else {
myFileList.add(file);
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "ActionSftpPut.Log.FilenameFromResult", filePrevious));
}
}
}
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "ActionSftpPut.Error.ArgFromPrevious"));
result.setNrErrors(1);
// free resource
myFileList = null;
return result;
}
}
if (copyingPreviousFiles) {
List<ResultFile> resultFiles = result.getResultFilesList();
if (resultFiles == null || resultFiles.size() == 0) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.ArgsFromPreviousNothingFiles"));
}
result.setResult(true);
return result;
}
try {
for (Iterator<ResultFile> it = resultFiles.iterator(); it.hasNext() && !parentWorkflow.isStopped(); ) {
ResultFile resultFile = it.next();
FileObject file = resultFile.getFile();
if (file != null) {
if (!file.exists()) {
logError(BaseMessages.getString(PKG, "ActionSftpPut.Log.FilefromPreviousNotFound", file.toString()));
} else {
myFileList.add(file);
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "ActionSftpPut.Log.FilenameFromResult", file.toString()));
}
}
}
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "ActionSftpPut.Error.ArgFromPrevious"));
result.setNrErrors(1);
// free resource
myFileList = null;
return result;
}
}
SftpClient sftpclient = null;
// String substitution..
String realServerName = resolve(serverName);
String realServerPort = resolve(serverPort);
String realUsername = resolve(userName);
String realPassword = Encr.decryptPasswordOptionallyEncrypted(resolve(password));
String realSftpDirString = resolve(remoteDirectory);
String realWildcard = resolve(wildcard);
String realLocalDirectory = resolve(localDirectory);
String realKeyFilename = null;
String realPassPhrase = null;
// Destination folder (Move to)
String realDestinationFolder = resolve(getDestinationFolder());
try {
if (getAfterFtps() == AFTER_FTPSPUT_MOVE) {
if (Utils.isEmpty(realDestinationFolder)) {
logError(BaseMessages.getString(PKG, "ActionSSH2PUT.Log.DestinatFolderMissing"));
result.setNrErrors(1);
return result;
} else {
FileObject folder = null;
try {
folder = HopVfs.getFileObject(realDestinationFolder);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(BaseMessages.getString(PKG, "ActionSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
result.setNrErrors(1);
return result;
}
}
realDestinationFolder = HopVfs.getFilename(folder);
} catch (Exception e) {
throw new HopException(e);
} finally {
if (folder != null) {
try {
folder.close();
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (isUseKeyFile()) {
// We must have here a private keyfilename
realKeyFilename = resolve(getKeyFilename());
if (Utils.isEmpty(realKeyFilename)) {
// Error..Missing keyfile
logError(BaseMessages.getString(PKG, "ActionSftp.Error.KeyFileMissing"));
result.setNrErrors(1);
return result;
}
if (!HopVfs.fileExists(realKeyFilename)) {
// Error.. can not reach keyfile
logError(BaseMessages.getString(PKG, "ActionSftp.Error.KeyFileNotFound"));
result.setNrErrors(1);
return result;
}
realPassPhrase = resolve(getKeyPassPhrase());
}
// Create sftp client to host ...
sftpclient = new SftpClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername, realKeyFilename, realPassPhrase);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername));
}
// Set compression
sftpclient.setCompression(getCompression());
// Set proxy?
String realProxyHost = resolve(getProxyHost());
if (!Utils.isEmpty(realProxyHost)) {
// Set proxy
sftpclient.setProxy(realProxyHost, resolve(getProxyPort()), resolve(getProxyUsername()), resolve(getProxyPassword()), getProxyType());
}
// login to ftp host ...
sftpclient.login(realPassword);
// move to spool dir ...
if (!Utils.isEmpty(realSftpDirString)) {
boolean existfolder = sftpclient.folderExists(realSftpDirString);
if (!existfolder) {
if (!isCreateRemoteFolder()) {
throw new HopException(BaseMessages.getString(PKG, "ActionSftpPut.Error.CanNotFindRemoteFolder", realSftpDirString));
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Error.CanNotFindRemoteFolder", realSftpDirString));
}
// Let's create folder
sftpclient.createFolder(realSftpDirString);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.RemoteFolderCreated", realSftpDirString));
}
}
sftpclient.chdir(realSftpDirString);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.ChangedDirectory", realSftpDirString));
}
}
if (!copyingPrevious && !copyingPreviousFiles) {
// Get all the files in the local directory...
myFileList = new ArrayList<>();
FileObject localFiles = HopVfs.getFileObject(realLocalDirectory);
FileObject[] children = localFiles.getChildren();
if (children != null) {
for (int i = 0; i < children.length; i++) {
// Get filename of file or directory
if (children[i].getType().equals(FileType.FILE)) {
myFileList.add(children[i]);
}
}
// end for
}
}
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.RowsFromPreviousResult", myFileList.size()));
}
Pattern pattern = null;
if (!copyingPrevious && !copyingPreviousFiles) {
if (!Utils.isEmpty(realWildcard)) {
pattern = Pattern.compile(realWildcard);
}
}
int nrFilesSent = 0;
int nrFilesMatched = 0;
// Get the files in the list and execute sftp.put() for each file
Iterator<FileObject> it = myFileList.iterator();
while (it.hasNext() && !parentWorkflow.isStopped()) {
FileObject myFile = it.next();
try {
String localFilename = myFile.toString();
String destinationFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(destinationFilename);
getIt = matcher.matches();
}
if (getIt) {
nrFilesMatched++;
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "ActionSftpPut.Log.PuttingFile", localFilename, realSftpDirString));
}
sftpclient.put(myFile, destinationFilename);
nrFilesSent++;
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.TransferredFile", localFilename));
}
// what's next ...
switch(getAfterFtps()) {
case AFTER_FTPSPUT_DELETE:
myFile.delete();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.DeletedFile", localFilename));
}
break;
case AFTER_FTPSPUT_MOVE:
FileObject destination = null;
try {
destination = HopVfs.getFileObject(realDestinationFolder + Const.FILE_SEPARATOR + myFile.getName().getBaseName());
myFile.moveTo(destination);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.FileMoved", myFile, destination));
}
} finally {
if (destination != null) {
destination.close();
}
}
break;
default:
if (addFilenameResut) {
// Add to the result files...
ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentWorkflow.getWorkflowName(), toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionSftpPut.Log.FilenameAddedToResultFilenames", localFilename));
}
}
break;
}
}
} finally {
if (myFile != null) {
myFile.close();
}
}
}
// end for
result.setResult(true);
// Abuse the nr of files retrieved vector in the results
//
result.setNrFilesRetrieved(nrFilesSent);
//
if (nrFilesMatched == 0) {
if (isSuccessWhenNoFile()) {
// Just warn user
if (isBasic()) {
logBasic(BaseMessages.getString(PKG, "ActionSftpPut.Error.NoFileToSend"));
}
} else {
// Fail
logError(BaseMessages.getString(PKG, "ActionSftpPut.Error.NoFileToSend"));
result.setNrErrors(1);
result.setResult(false);
return result;
}
}
} catch (Exception e) {
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "ActionSftpPut.Exception", e.getMessage()));
logError(Const.getStackTracker(e));
} finally {
// close connection, if possible
try {
if (sftpclient != null) {
sftpclient.disconnect();
}
} catch (Exception e) {
// just ignore this, makes no big difference
}
// end catch
myFileList = null;
}
return result;
}
use of org.apache.hop.core.exception.HopException in project hop by apache.
the class ActionFtpDelete method execute.
/**
* Needed for the Vector coming from sshclient.ls() *
*/
@Override
@SuppressWarnings("unchecked")
public Result execute(Result previousResult, int nr) {
log.logBasic(BaseMessages.getString(PKG, "ActionFTPDelete.Started", serverName));
RowMetaAndData resultRow = null;
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
result.setResult(false);
nrErrors = 0;
nrFilesDeleted = 0;
successConditionBroken = false;
HashSet<String> listPreviousFiles = new HashSet<>();
// Here let's put some controls before stating the workflow
String realServerName = resolve(serverName);
String realServerPassword = Utils.resolvePassword(this, password);
String realFtpDirectory = resolve(remoteDirectory);
int realServerPort = Const.toInt(resolve(serverPort), 0);
String realUsername = resolve(userName);
String realPassword = Utils.resolvePassword(this, password);
String realProxyHost = resolve(proxyHost);
String realProxyUsername = resolve(proxyUsername);
String realProxyPassword = Utils.resolvePassword(this, proxyPassword);
int realProxyPort = Const.toInt(resolve(proxyPort), 0);
String realKeyFilename = resolve(keyFilename);
String realKeyPass = resolve(keyFilePass);
// The following is used to apply a path for SSH because the SFTPv3Client doesn't let us
// specify/change dirs
//
String sourceFolder = "";
if (isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionFTPDelete.Start"));
}
if (copyPrevious && rows.size() == 0) {
if (isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "ActionFTPDelete.ArgsFromPreviousNothing"));
}
result.setResult(true);
return result;
}
try {
// Get all the files in the current directory...
String[] filelist = null;
if (protocol.equals(PROTOCOL_FTP)) {
// establish the connection
ftpConnect(realFtpDirectory);
filelist = ftpclient.listNames();
// CHECK THIS !!!
if (filelist.length == 1) {
String translatedWildcard = resolve(wildcard);
if (!Utils.isEmpty(translatedWildcard)) {
if (filelist[0].startsWith(translatedWildcard)) {
throw new HopException(filelist[0]);
}
}
}
} else if (protocol.equals(PROTOCOL_SFTP)) {
// establish the secure connection
sftpConnect(realServerName, realUsername, realServerPort, realPassword, realFtpDirectory);
// Get all the files in the current directory...
filelist = sftpclient.dir();
}
if (isDetailed()) {
logDetailed("ActionFTPDelete.FoundNFiles", String.valueOf(filelist.length));
}
int found = filelist == null ? 0 : filelist.length;
if (found == 0) {
result.setResult(true);
return result;
}
Pattern pattern = null;
if (copyPrevious) {
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size(); iteration++) {
resultRow = rows.get(iteration);
// Get file names
String filePrevious = resultRow.getString(0, null);
if (!Utils.isEmpty(filePrevious)) {
listPreviousFiles.add(filePrevious);
}
}
} else {
if (!Utils.isEmpty(wildcard)) {
String realWildcard = resolve(wildcard);
pattern = Pattern.compile(realWildcard);
}
}
if (!getSuccessCondition().equals(SUCCESS_IF_ALL_FILES_DOWNLOADED)) {
limitFiles = Const.toInt(resolve(getLimitSuccess()), 10);
}
// Get the files in the list...
for (int i = 0; i < filelist.length && !parentWorkflow.isStopped(); i++) {
if (successConditionBroken) {
throw new Exception(BaseMessages.getString(PKG, "ActionFTPDelete.SuccesConditionBroken"));
}
boolean getIt = false;
if (isDebug()) {
logDebug(BaseMessages.getString(PKG, "ActionFTPDelete.AnalysingFile", filelist[i]));
}
try {
// First see if the file matches the regular expression!
if (copyPrevious) {
if (listPreviousFiles.contains(filelist[i])) {
getIt = true;
}
} else {
if (pattern != null) {
Matcher matcher = pattern.matcher(filelist[i]);
getIt = matcher.matches();
}
}
if (getIt) {
// Delete file
if (protocol.equals(PROTOCOL_FTP)) {
ftpclient.deleteFile(filelist[i]);
} else if (protocol.equals(PROTOCOL_SFTP)) {
sftpclient.delete(filelist[i]);
}
if (isDetailed()) {
logDetailed("ActionFTPDelete.RemotefileDeleted", filelist[i]);
}
updateDeletedFiles();
}
} catch (Exception e) {
// Update errors number
updateErrors();
logError(BaseMessages.getString(PKG, "ActionFtp.UnexpectedError", e.getMessage()));
if (successConditionBroken) {
throw new Exception(BaseMessages.getString(PKG, "ActionFTPDelete.SuccesConditionBroken"));
}
}
}
// end for
} catch (Exception e) {
updateErrors();
logError(BaseMessages.getString(PKG, "ActionFTPDelete.ErrorGetting", e.getMessage()));
logError(Const.getStackTracker(e));
} finally {
if (ftpclient != null && ftpclient.isConnected()) {
try {
ftpclient.quit();
ftpclient = null;
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "ActionFTPDelete.ErrorQuitting", e.getMessage()));
}
}
if (sftpclient != null) {
try {
sftpclient.disconnect();
sftpclient = null;
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "ActionFTPDelete.ErrorQuitting", e.getMessage()));
}
}
FtpClientUtil.clearSocksJvmSettings();
}
result.setResult(!successConditionBroken);
result.setNrFilesRetrieved(nrFilesDeleted);
result.setNrErrors(nrErrors);
return result;
}
use of org.apache.hop.core.exception.HopException in project hop by apache.
the class ActionEvalTableContentDialog method getSql.
private void getSql() {
DatabaseMeta inf = getWorkflowMeta().findDatabase(wConnection.getText());
if (inf != null) {
DatabaseExplorerDialog std = new DatabaseExplorerDialog(shell, SWT.NONE, variables, inf, getWorkflowMeta().getDatabases());
if (std.open()) {
String sql = "SELECT *" + Const.CR + "FROM " + inf.getQuotedSchemaTableCombination(variables, std.getSchemaName(), std.getTableName()) + Const.CR;
wSql.setText(sql);
MessageBox yn = new MessageBox(shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
yn.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.IncludeFieldNamesInSQL"));
yn.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionQuestion"));
int id = yn.open();
switch(id) {
case SWT.CANCEL:
break;
case SWT.NO:
wSql.setText(sql);
break;
case SWT.YES:
Database db = new Database(loggingObject, variables, inf);
try {
db.connect();
IRowMeta fields = db.getQueryFields(sql, false);
if (fields != null) {
sql = "SELECT" + Const.CR;
for (int i = 0; i < fields.size(); i++) {
IValueMeta field = fields.getValueMeta(i);
if (i == 0) {
sql += " ";
} else {
sql += ", ";
}
sql += inf.quoteField(field.getName()) + Const.CR;
}
sql += "FROM " + inf.getQuotedSchemaTableCombination(variables, std.getSchemaName(), std.getTableName()) + Const.CR;
wSql.setText(sql);
} else {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.ERROR_CouldNotRetrieveFields") + Const.CR + BaseMessages.getString(PKG, "ActionEvalTableContent.PerhapsNoPermissions"));
mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError2"));
mb.open();
}
} catch (HopException e) {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError3"));
mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.AnErrorOccurred") + Const.CR + e.getMessage());
mb.open();
} finally {
db.disconnect();
}
break;
default:
break;
}
}
} else {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.ConnectionNoLongerAvailable"));
mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError4"));
mb.open();
}
}
Aggregations