use of org.pentaho.di.trans.TransAdapter in project pentaho-kettle by pentaho.
the class TransformationResource method addTransformation.
@PUT
@Path("/add")
@Produces({ MediaType.APPLICATION_JSON })
public TransformationStatus addTransformation(String xml) {
TransConfiguration transConfiguration;
try {
transConfiguration = TransConfiguration.fromXML(xml.toString());
TransMeta transMeta = transConfiguration.getTransMeta();
TransExecutionConfiguration transExecutionConfiguration = transConfiguration.getTransExecutionConfiguration();
transMeta.setLogLevel(transExecutionConfiguration.getLogLevel());
LogChannelInterface log = CarteSingleton.getInstance().getLog();
if (log.isDetailed()) {
log.logDetailed("Logging level set to " + log.getLogLevel().getDescription());
}
transMeta.injectVariables(transExecutionConfiguration.getVariables());
// Also copy the parameters over...
//
Map<String, String> params = transExecutionConfiguration.getParams();
for (String param : params.keySet()) {
String value = params.get(param);
transMeta.setParameterValue(param, value);
}
// If there was a repository, we know about it at this point in time.
//
TransExecutionConfiguration executionConfiguration = transConfiguration.getTransExecutionConfiguration();
final Repository repository = transConfiguration.getTransExecutionConfiguration().getRepository();
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(getClass().getName(), LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
servletLoggingObject.setLogLevel(executionConfiguration.getLogLevel());
// Create the transformation and store in the list...
//
final Trans trans = new Trans(transMeta, servletLoggingObject);
trans.setRepository(repository);
trans.setSocketRepository(CarteSingleton.getInstance().getSocketRepository());
CarteSingleton.getInstance().getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
trans.setContainerObjectId(carteObjectId);
if (repository != null) {
// The repository connection is open: make sure we disconnect from the repository once we
// are done with this transformation.
//
trans.addTransListener(new TransAdapter() {
public void transFinished(Trans trans) {
repository.disconnect();
}
});
}
return getTransformationStatus(carteObjectId);
} catch (KettleException e) {
e.printStackTrace();
}
return null;
}
use of org.pentaho.di.trans.TransAdapter in project pentaho-kettle by pentaho.
the class AddTransServlet method doGet.
/**
* <div id="mindtouch">
* <h1>/kettle/addTrans</h1>
* <a name="POST"></a>
* <h2>POST</h2>
* <p>Uploads and executes transformation configuration XML file.
* Uploads xml file containing transformation and transformation_execution_configuration
* (wrapped in transformation_configuration tag) to be executed and executes it. Method relies
* on the input parameter to determine if xml or html reply should be produced. The transformation_configuration xml is
* transferred within request body.
*
* <code>transformation name of the executed transformation </code> will be returned in the Response object
* or <code>message</code> describing error occurred. To determine if the call successful or not you should
* rely on <code>result</code> parameter in response.</p>
*
* <p><b>Example Request:</b><br />
* <pre function="syntax.xml">
* POST /kettle/addTrans/?xml=Y
* </pre>
* <p>Request body should contain xml containing transformation_configuration (transformation and
* transformation_execution_configuration wrapped in transformation_configuration tag).</p>
* </p>
* <h3>Parameters</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>xml</td>
* <td>Boolean flag set to either <code>Y</code> or <code>N</code> describing if xml or html reply
* should be produced.</td>
* <td>boolean, optional</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">element:</td>
* <td>(custom)</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml, text/html</td>
* </tr>
* </tbody>
* </table>
* <p>Response wraps transformation name that was executed or error stack trace
* if an error occurred. Response has <code>result</code> OK if there were no errors. Otherwise it returns ERROR.</p>
*
* <p><b>Example Response:</b></p>
* <pre function="syntax.xml">
* <?xml version="1.0" encoding="UTF-8"?>
* <webresult>
* <result>OK</result>
* <message>Transformation 'dummy-trans' was added to Carte with id eb4a92ff-6852-4307-9f74-3c74bd61f829</message>
* <id>eb4a92ff-6852-4307-9f74-3c74bd61f829</id>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed and XML response is returned.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
*</table>
*</div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug("Addition of transformation requested");
}
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
PrintWriter out = response.getWriter();
BufferedReader in = request.getReader();
if (log.isDetailed()) {
logDetailed("Encoding: " + request.getCharacterEncoding());
}
if (useXML) {
response.setContentType("text/xml");
out.print(XMLHandler.getXMLHeader());
} else {
response.setContentType("text/html");
out.println("<HTML>");
out.println("<HEAD><TITLE>Add transformation</TITLE></HEAD>");
out.println("<BODY>");
}
response.setStatus(HttpServletResponse.SC_OK);
String realLogFilename = null;
TransExecutionConfiguration transExecutionConfiguration = null;
try {
// First read the complete transformation in memory from the request
//
StringBuilder xml = new StringBuilder(request.getContentLength());
int c;
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a transformation configuration
//
TransConfiguration transConfiguration = TransConfiguration.fromXML(xml.toString());
TransMeta transMeta = transConfiguration.getTransMeta();
transExecutionConfiguration = transConfiguration.getTransExecutionConfiguration();
transMeta.setLogLevel(transExecutionConfiguration.getLogLevel());
if (log.isDetailed()) {
logDetailed("Logging level set to " + log.getLogLevel().getDescription());
}
transMeta.injectVariables(transExecutionConfiguration.getVariables());
// Also copy the parameters over...
//
Map<String, String> params = transExecutionConfiguration.getParams();
for (String param : params.keySet()) {
String value = params.get(param);
transMeta.setParameterValue(param, value);
}
// If there was a repository, we know about it at this point in time.
//
final Repository repository = transExecutionConfiguration.getRepository();
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
servletLoggingObject.setLogLevel(transExecutionConfiguration.getLogLevel());
// Create the transformation and store in the list...
//
final Trans trans = new Trans(transMeta, servletLoggingObject);
if (transExecutionConfiguration.isSetLogfile()) {
realLogFilename = transExecutionConfiguration.getLogFileName();
final LogChannelFileWriter logChannelFileWriter;
try {
FileUtil.createParentFolder(AddTransServlet.class, realLogFilename, transExecutionConfiguration.isCreateParentFolder(), trans.getLogChannel(), trans);
logChannelFileWriter = new LogChannelFileWriter(servletLoggingObject.getLogChannelId(), KettleVFS.getFileObject(realLogFilename), transExecutionConfiguration.isSetAppendLogfile());
logChannelFileWriter.startLogging();
trans.addTransListener(new TransAdapter() {
@Override
public void transFinished(Trans trans) throws KettleException {
if (logChannelFileWriter != null) {
logChannelFileWriter.stopLogging();
}
}
});
} catch (KettleException e) {
logError(Const.getStackTracker(e));
}
}
trans.setRepository(repository);
trans.setSocketRepository(getSocketRepository());
getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
trans.setContainerObjectId(carteObjectId);
if (repository != null) {
// The repository connection is open: make sure we disconnect from the repository once we
// are done with this transformation.
//
trans.addTransListener(new TransAdapter() {
public void transFinished(Trans trans) {
repository.disconnect();
}
});
}
String message = "Transformation '" + trans.getName() + "' was added to Carte with id " + carteObjectId;
if (useXML) {
// Return the log channel id as well
//
out.println(new WebResult(WebResult.STRING_OK, message, carteObjectId));
} else {
out.println("<H1>" + message + "</H1>");
out.println("<p><a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + trans.getName() + "&id=" + carteObjectId + "\">Go to the transformation status page</a><p>");
}
} catch (Exception ex) {
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
} else {
out.println("<p>");
out.println("<pre>");
ex.printStackTrace(out);
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
use of org.pentaho.di.trans.TransAdapter in project pentaho-kettle by pentaho.
the class BaseJobServlet method createTrans.
protected Trans createTrans(TransConfiguration transConfiguration) throws UnknownParamException {
TransMeta transMeta = transConfiguration.getTransMeta();
TransExecutionConfiguration transExecutionConfiguration = transConfiguration.getTransExecutionConfiguration();
transMeta.setLogLevel(transExecutionConfiguration.getLogLevel());
transMeta.injectVariables(transExecutionConfiguration.getVariables());
// Also copy the parameters over...
copyParameters(transMeta, transExecutionConfiguration.getParams());
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = getServletLogging(carteObjectId, transExecutionConfiguration.getLogLevel());
// Create the transformation and store in the list...
final Trans trans = new Trans(transMeta, servletLoggingObject);
trans.setMetaStore(transformationMap.getSlaveServerConfig().getMetaStore());
if (transExecutionConfiguration.isSetLogfile()) {
String realLogFilename = transExecutionConfiguration.getLogFileName();
try {
FileUtil.createParentFolder(AddTransServlet.class, realLogFilename, transExecutionConfiguration.isCreateParentFolder(), trans.getLogChannel(), trans);
final LogChannelFileWriter logChannelFileWriter = new LogChannelFileWriter(servletLoggingObject.getLogChannelId(), KettleVFS.getFileObject(realLogFilename), transExecutionConfiguration.isSetAppendLogfile());
logChannelFileWriter.startLogging();
trans.addTransListener(new TransAdapter() {
@Override
public void transFinished(Trans trans) throws KettleException {
if (logChannelFileWriter != null) {
logChannelFileWriter.stopLogging();
}
}
});
} catch (KettleException e) {
logError(Const.getStackTracker(e));
}
}
// If there was a repository, we know about it at this point in time.
final Repository repository = transExecutionConfiguration.getRepository();
trans.setRepository(repository);
trans.setSocketRepository(getSocketRepository());
trans.setContainerObjectId(carteObjectId);
getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
if (repository != null) {
// The repository connection is open: make sure we disconnect from the repository once we
// are done with this transformation.
trans.addTransListener(new TransAdapter() {
public void transFinished(Trans trans) {
repository.disconnect();
}
});
}
final Long passedBatchId = transExecutionConfiguration.getPassedBatchId();
if (passedBatchId != null) {
trans.setPassedBatchId(passedBatchId);
}
return trans;
}
use of org.pentaho.di.trans.TransAdapter in project pentaho-kettle by pentaho.
the class ExecuteTransServlet method doGet.
/**
* <div id="mindtouch">
* <h1>/kettle/executeTrans</h1>
* <a name="GET"></a>
* <h2>GET</h2>
* <p>Executes transformation from the specified repository.
* Connects to the repository provided as a parameter, loads the transformation from it and executes it.
* Empty response is returned or response contains output of an error happened during the transformation execution.
* Response contains <code>ERROR</code> result if error happened during transformation execution.</p>
*
* <p><b>Example Request:</b><br />
* <pre function="syntax.xml">
* GET /kettle/executeTrans/?rep=my_repository&user=my_user&pass=my_password&trans=my_trans&level=INFO
* </pre>
*
* </p>
* <h3>Parameters</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>rep</td>
* <td>Repository id to connect to.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>user</td>
* <td>User name to be used to connect to repository.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>pass</td>
* <td>User password to be used to connect to repository.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>trans</td>
* <td>Transfromation name to be loaded and executed.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>level</td>
* <td>Logging level to be used for transformation execution (i.e. Debug).</td>
* <td>query</td>
* </tr>
* <tr>
* <td>*any name*</td>
* <td>All the other parameters will be sent to the transformation for using as variables.
* When necessary you can add custom parameters to the request.
* They will be used to set the transformation variables values..</td>
* <td>query</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">element:</td>
* <td>(custom)</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>application/xml</td>
* </tr>
* </tbody>
* </table>
* <p>Response contains error output of the transformation executed or nothing
* if the execution was successful.</p>
*
* <p><b>Example Error Response:</b></p>
* <pre function="syntax.xml">
* <webresult>
* <result>ERROR</result>
* <message>Unexpected error executing the transformation:
* 
org.pentaho.di.core.exception.KettleException:
* 
Unable to find transformation 'dummy-trans.ktr' in directory
* :/home/admin

 at
* org.pentaho.di.www.ExecuteTransServlet.loadTransformation(ExecuteTransServlet.java:214)

* at org.pentaho.di.www.ExecuteTransServlet.doGet(ExecuteTransServlet.java:104)

* at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)

* at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)

* at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)

* at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:390)

* at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)

* at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)

* at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)

* at org.mortbay.jetty.handler.HandlerCollection.handle(HandlerCollection.java:114)

* at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)

* at org.mortbay.jetty.Server.handle(Server.java:326)

* at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:536)

* at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:915)

* at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:539)

* at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)

* at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:405)

* at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)

* at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

* </message>
* <id/>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
*</table>
*</div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "ExecuteTransServlet.Log.ExecuteTransRequested"));
}
// Options taken from PAN
//
String[] knownOptions = new String[] { "rep", "user", "pass", "trans", "level" };
String repOption = request.getParameter("rep");
String userOption = request.getParameter("user");
String passOption = Encr.decryptPasswordOptionallyEncrypted(request.getParameter("pass"));
String transOption = request.getParameter("trans");
String levelOption = request.getParameter("level");
response.setStatus(HttpServletResponse.SC_OK);
String encoding = System.getProperty("KETTLE_DEFAULT_SERVLET_ENCODING", null);
if (encoding != null && !Utils.isEmpty(encoding.trim())) {
response.setCharacterEncoding(encoding);
response.setContentType("text/html; charset=" + encoding);
}
PrintWriter out = response.getWriter();
try {
final Repository repository = openRepository(repOption, userOption, passOption);
final TransMeta transMeta = loadTransformation(repository, transOption);
// Set the servlet parameters as variables in the transformation
//
String[] parameters = transMeta.listParameters();
Enumeration<?> parameterNames = request.getParameterNames();
while (parameterNames.hasMoreElements()) {
String parameter = (String) parameterNames.nextElement();
String[] values = request.getParameterValues(parameter);
//
if (Const.indexOfString(parameter, knownOptions) < 0) {
//
if (Const.indexOfString(parameter, parameters) < 0) {
transMeta.setVariable(parameter, values[0]);
} else {
transMeta.setParameterValue(parameter, values[0]);
}
}
}
TransExecutionConfiguration transExecutionConfiguration = new TransExecutionConfiguration();
LogLevel logLevel = LogLevel.getLogLevelForCode(levelOption);
transExecutionConfiguration.setLogLevel(logLevel);
TransConfiguration transConfiguration = new TransConfiguration(transMeta, transExecutionConfiguration);
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
servletLoggingObject.setLogLevel(logLevel);
// Create the transformation and store in the list...
//
final Trans trans = new Trans(transMeta, servletLoggingObject);
trans.setRepository(repository);
trans.setSocketRepository(getSocketRepository());
getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
trans.setContainerObjectId(carteObjectId);
if (repository != null) {
// The repository connection is open: make sure we disconnect from the repository once we
// are done with this transformation.
//
trans.addTransListener(new TransAdapter() {
public void transFinished(Trans trans) {
repository.disconnect();
}
});
}
// Pass the servlet print writer to the transformation...
//
trans.setServletPrintWriter(out);
trans.setServletReponse(response);
trans.setServletRequest(request);
try {
// Execute the transformation...
//
executeTrans(trans);
out.flush();
} catch (Exception executionException) {
String logging = KettleLogStore.getAppender().getBuffer(trans.getLogChannelId(), false).toString();
throw new KettleException("Error executing transformation: " + logging, executionException);
}
} catch (Exception ex) {
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "ExecuteTransServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
}
}
use of org.pentaho.di.trans.TransAdapter in project pentaho-kettle by pentaho.
the class TransPreviewDelegate method capturePreviewData.
public void capturePreviewData(final Trans trans, List<StepMeta> stepMetas) {
final StringBuffer loggingText = new StringBuffer();
// First clean out previous preview data. Otherwise this method leaks memory like crazy.
//
previewLogMap.clear();
previewMetaMap.clear();
previewDataMap.clear();
try {
final TransMeta transMeta = trans.getTransMeta();
for (final StepMeta stepMeta : stepMetas) {
final RowMetaInterface rowMeta = transMeta.getStepFields(stepMeta).clone();
previewMetaMap.put(stepMeta, rowMeta);
final List<Object[]> rowsData;
if (previewMode == PreviewMode.LAST) {
rowsData = new LinkedList<Object[]>();
} else {
rowsData = new ArrayList<Object[]>();
}
previewDataMap.put(stepMeta, rowsData);
previewLogMap.put(stepMeta, loggingText);
StepInterface step = trans.findRunThread(stepMeta.getName());
if (step != null) {
switch(previewMode) {
case LAST:
step.addRowListener(new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
try {
rowsData.add(rowMeta.cloneRow(row));
if (rowsData.size() > PropsUI.getInstance().getDefaultPreviewSize()) {
rowsData.remove(0);
}
} catch (Exception e) {
throw new KettleStepException("Unable to clone row for metadata : " + rowMeta, e);
}
}
});
break;
default:
step.addRowListener(new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
if (rowsData.size() < PropsUI.getInstance().getDefaultPreviewSize()) {
try {
rowsData.add(rowMeta.cloneRow(row));
} catch (Exception e) {
throw new KettleStepException("Unable to clone row for metadata : " + rowMeta, e);
}
}
}
});
break;
}
}
}
} catch (Exception e) {
loggingText.append(Const.getStackTracker(e));
}
// In case there were errors during preview...
//
trans.addTransListener(new TransAdapter() {
@Override
public void transFinished(Trans trans) throws KettleException {
//
if (trans.getErrors() != 0) {
//
for (StepMetaDataCombi combi : trans.getSteps()) {
if (combi.copy == 0) {
StringBuffer logBuffer = KettleLogStore.getAppender().getBuffer(combi.step.getLogChannel().getLogChannelId(), false);
previewLogMap.put(combi.stepMeta, logBuffer);
}
}
}
}
});
}
Aggregations