use of org.pentaho.di.trans.step.StepInterface in project data-access by pentaho.
the class StagingTransformGenerator method loadTable.
/**
* Stages the data from a CSV file into a database table. As the table is loading, a {@link TransformStats} monitors
* the progress. This is placed in the supplied {@link IPentahoSession} to allow interrogation under the attribute key
* <code>FileTransformStats_<em>fileName</em></code>
*
* @param truncate
* @param session
* @throws CsvTransformGeneratorException
*/
public void loadTable(boolean truncate, IPentahoSession session, boolean async) throws CsvTransformGeneratorException {
if (session == null) {
// $NON-NLS-1$
throw new IllegalArgumentException("IPentahoSession cannot be null");
}
if (tableName == null) {
// $NON-NLS-1$
throw new IllegalArgumentException("Table name cannot be null");
}
if (transformStats != null) {
transformStats.setRowsFinished(false);
transformStats.setRowsStarted(true);
transformStats.setTotalRecords(0);
transformStats.setRowsRejected(0);
}
Trans trans = createTransform(true);
// the table output is the last step
StepMeta[] steps = trans.getTransMeta().getStepsArray();
StepMeta tableStepMeta = steps[steps.length - 1];
TableOutputMeta meta = (TableOutputMeta) tableStepMeta.getStepMetaInterface();
meta.setDatabaseMeta(targetDatabaseMeta);
meta.setTruncateTable(truncate);
try {
prepareTransform(trans, session);
} catch (Exception e) {
// $NON-NLS-1$
error("Preview Failed: transformation preparation", e);
Throwable e2 = e.getCause();
e2 = e2 == null ? e : e2;
throw new CsvTransformGeneratorException("Preview Failed: transformation preparation: loadTable", e2, // $NON-NLS-1$
getStackTraceAsString(e2));
}
StepInterface step = trans.findRunThread(TABLE_OUTPUT);
PdiTransListener listener = new PdiTransListener(trans, step, transformStats);
// start the listener in a thread
Thread listenerThread = new Thread(listener);
listenerThread.start();
session.setAttribute(TRANS_SESSION_ATTR, trans);
if (async) {
executeTransformAsync(trans);
} else {
executeTransformSync(trans, null, session);
}
}
use of org.pentaho.di.trans.step.StepInterface in project pentaho-metaverse by pentaho.
the class StepExternalResourceConsumerListener method callExtensionPoint.
/**
* This method is called by the Kettle code when a step is about to start
*
* @param log the logging channel to log debugging information to
* @param object The subject object that is passed to the plugin code
* @throws org.pentaho.di.core.exception.KettleException In case the plugin decides that an error has occurred
* and the parent process should stop.
*/
@Override
public void callExtensionPoint(LogChannelInterface log, Object object) throws KettleException {
if (stepConsumerProvider == null) {
stepConsumerProvider = (IStepExternalResourceConsumerProvider) MetaverseBeanUtil.getInstance().get("IStepExternalResourceConsumerProvider");
}
StepMetaDataCombi stepCombi = (StepMetaDataCombi) object;
if (stepCombi != null) {
StepMetaInterface meta = stepCombi.meta;
StepInterface step = stepCombi.step;
if (meta != null) {
Class<?> metaClass = meta.getClass();
if (BaseStepMeta.class.isAssignableFrom(metaClass)) {
if (stepConsumerProvider != null) {
// Put the class into a collection and get the consumers that can process this class
Set<Class<?>> metaClassSet = new HashSet<Class<?>>(1);
metaClassSet.add(metaClass);
List<IStepExternalResourceConsumer> stepConsumers = stepConsumerProvider.getExternalResourceConsumers(metaClassSet);
if (stepConsumers != null) {
for (IStepExternalResourceConsumer stepConsumer : stepConsumers) {
// We might know enough at this point, so call the consumer
Collection<IExternalResourceInfo> resources = stepConsumer.getResourcesFromMeta(meta);
addExternalResources(resources, step);
// Add a RowListener if the step is data-driven
if (stepConsumer.isDataDriven(meta)) {
stepCombi.step.addRowListener(new StepExternalConsumerRowListener(stepConsumer, step));
}
}
}
}
}
}
}
}
use of org.pentaho.di.trans.step.StepInterface in project pentaho-metaverse by pentaho.
the class StepExternalResourceConsumerListenerTest method testCallStepAddExternalResources.
@Test
public void testCallStepAddExternalResources() {
StepExternalResourceConsumerListener stepExtensionPoint = new StepExternalResourceConsumerListener();
stepExtensionPoint.addExternalResources(null, null);
StepInterface mockStep = mock(StepInterface.class);
Trans mockTrans = mock(Trans.class);
when(mockStep.getTrans()).thenReturn(mockTrans);
IExecutionProfile executionProfile = mock(IExecutionProfile.class);
IExecutionData executionData = mock(IExecutionData.class);
when(executionProfile.getExecutionData()).thenReturn(executionData);
LineageHolder holder = new LineageHolder();
holder.setExecutionProfile(executionProfile);
TransLineageHolderMap.getInstance().putLineageHolder(mockTrans, holder);
Collection<IExternalResourceInfo> externalResources = new ArrayList<IExternalResourceInfo>();
stepExtensionPoint.addExternalResources(externalResources, mockStep);
IExternalResourceInfo externalResource = mock(IExternalResourceInfo.class);
externalResources.add(externalResource);
stepExtensionPoint.addExternalResources(externalResources, mockStep);
}
use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.
the class GetTransStatusServlet method doGet.
/**
* <div id="mindtouch">
* <h1>/kettle/transStatus</h1>
* <a name="GET"></a>
* <h2>GET</h2>
* <p>Retrieves status of the specified transformation. Status is returned as HTML or XML output
* depending on the input parameters. Status contains information about last execution of the transformation.</p>
* <p><b>Example Request:</b><br />
* <pre function="syntax.xml">
* GET /kettle/transStatus/?name=dummy-trans&xml=Y
* </pre>
*
* </p>
* <h3>Parameters</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>name</td>
* <td>Name of the transformation to be used for status generation.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>xml</td>
* <td>Boolean flag which defines output format <code>Y</code> forces XML output to be generated.
* HTML is returned otherwise.</td>
* <td>boolean, optional</td>
* </tr>
* <tr>
* <td>id</td>
* <td>Carte id of the transformation to be used for status generation.</td>
* <td>query, optional</td>
* </tr>
* <tr>
* <td>from</td>
* <td>Start line number of the execution log to be included into response.</td>
* <td>integer, optional</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">element:</td>
* <td>(custom)</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml, text/html</td>
* </tr>
* </tbody>
* </table>
* <p> Response XML or HTML response containing details about the transformation specified.
* If an error occurs during method invocation <code>result</code> field of the response
* will contain <code>ERROR</code> status.</p>
*
* <p><b>Example Response:</b></p>
* <pre function="syntax.xml">
* <?xml version="1.0" encoding="UTF-8"?>
* <transstatus>
* <transname>dummy-trans</transname>
* <id>c56961b2-c848-49b8-abde-76c8015e29b0</id>
* <status_desc>Stopped</status_desc>
* <error_desc/>
* <paused>N</paused>
* <stepstatuslist>
* <stepstatus><stepname>Dummy (do nothing)</stepname>
* <copy>0</copy><linesRead>0</linesRead>
* <linesWritten>0</linesWritten><linesInput>0</linesInput>
* <linesOutput>0</linesOutput><linesUpdated>0</linesUpdated>
* <linesRejected>0</linesRejected><errors>0</errors>
* <statusDescription>Stopped</statusDescription><seconds>0.0</seconds>
* <speed>-</speed><priority>-</priority><stopped>Y</stopped>
* <paused>N</paused>
* </stepstatus>
* </stepstatuslist>
* <first_log_line_nr>0</first_log_line_nr>
* <last_log_line_nr>37</last_log_line_nr>
* <result>
* <lines_input>0</lines_input>
* <lines_output>0</lines_output>
* <lines_read>0</lines_read>
* <lines_written>0</lines_written>
* <lines_updated>0</lines_updated>
* <lines_rejected>0</lines_rejected>
* <lines_deleted>0</lines_deleted>
* <nr_errors>0</nr_errors>
* <nr_files_retrieved>0</nr_files_retrieved>
* <entry_nr>0</entry_nr>
* <result>Y</result>
* <exit_status>0</exit_status>
* <is_stopped>Y</is_stopped>
* <log_channel_id>10e2c832-07da-409a-a5ba-4b90a234e957</log_channel_id>
* <log_text/>
* <result-file></result-file>
* <result-rows></result-rows>
* </result>
* <logging_string><![CDATA[H4sIAAAAAAAAADMyMDTRNzTUNzJRMDSyMrC0MjFV0FVIKc3NrdQtKUrMKwbyXDKLCxJLkjMy89IViksSi0pSUxTS8osUwPJARm5iSWZ+nkI0kq5YXi4AQVH5bFoAAAA=]]></logging_string>
* </transstatus>
* </pre>
*
* <h3>Status Codes</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
* </table>
* </div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "TransStatusServlet.Log.TransStatusRequested"));
}
String transName = request.getParameter("name");
String id = request.getParameter("id");
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
int startLineNr = Const.toInt(request.getParameter("from"), 0);
response.setStatus(HttpServletResponse.SC_OK);
if (useXML) {
response.setContentType("text/xml");
response.setCharacterEncoding(Const.XML_ENCODING);
} else {
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html;charset=UTF-8");
}
// ID is optional...
//
Trans trans;
CarteObjectEntry entry;
if (Utils.isEmpty(id)) {
// get the first transformation that matches...
//
entry = getTransformationMap().getFirstCarteObjectEntry(transName);
if (entry == null) {
trans = null;
} else {
id = entry.getId();
trans = getTransformationMap().getTransformation(entry);
}
} else {
// Take the ID into account!
//
entry = new CarteObjectEntry(transName, id);
trans = getTransformationMap().getTransformation(entry);
}
if (trans != null) {
if (useXML) {
try {
OutputStream out = null;
byte[] data = null;
String logId = trans.getLogChannelId();
boolean finishedOrStopped = trans.isFinishedOrStopped();
boolean sendResultXmlWithStatus = "Y".equalsIgnoreCase(request.getParameter(SEND_RESULT));
boolean dontUseCache = sendResultXmlWithStatus;
if (finishedOrStopped && (data = cache.get(logId, startLineNr)) != null && !dontUseCache) {
response.setContentLength(XML_HEADER.length + data.length);
out = response.getOutputStream();
out.write(XML_HEADER);
out.write(data);
out.flush();
} else {
int lastLineNr = KettleLogStore.getLastBufferLineNr();
String logText = getLogText(trans, startLineNr, lastLineNr);
response.setContentType("text/xml");
response.setCharacterEncoding(Const.XML_ENCODING);
SlaveServerTransStatus transStatus = new SlaveServerTransStatus(transName, entry.getId(), trans.getStatus());
transStatus.setFirstLoggingLineNr(startLineNr);
transStatus.setLastLoggingLineNr(lastLineNr);
transStatus.setLogDate(trans.getLogDate());
for (int i = 0; i < trans.nrSteps(); i++) {
StepInterface baseStep = trans.getRunThread(i);
if ((baseStep.isRunning()) || baseStep.getStatus() != StepExecutionStatus.STATUS_EMPTY) {
StepStatus stepStatus = new StepStatus(baseStep);
transStatus.getStepStatusList().add(stepStatus);
}
}
// The log can be quite large at times, we are going to putIfAbsent a base64 encoding around a compressed
// stream
// of bytes to handle this one.
String loggingString = HttpUtil.encodeBase64ZippedString(logText);
transStatus.setLoggingString(loggingString);
// transStatus.setLoggingUncompressedSize( logText.length() );
// Also set the result object...
//
transStatus.setResult(trans.getResult());
// Is the transformation paused?
//
transStatus.setPaused(trans.isPaused());
// Send the result back as XML
//
String xml = transStatus.getXML(sendResultXmlWithStatus);
data = xml.getBytes(Charset.forName(Const.XML_ENCODING));
out = response.getOutputStream();
response.setContentLength(XML_HEADER.length + data.length);
out.write(XML_HEADER);
out.write(data);
out.flush();
if (finishedOrStopped && logId != null && !dontUseCache) {
cache.put(logId, xml, startLineNr);
}
}
response.flushBuffer();
} catch (KettleException e) {
throw new ServletException("Unable to get the transformation status in XML format", e);
}
} else {
PrintWriter out = response.getWriter();
int lastLineNr = KettleLogStore.getLastBufferLineNr();
response.setContentType("text/html;charset=UTF-8");
out.println("<HTML>");
out.println("<HEAD>");
out.println("<TITLE>" + BaseMessages.getString(PKG, "TransStatusServlet.KettleTransStatus") + "</TITLE>");
if (EnvUtil.getSystemProperty(Const.KETTLE_CARTE_REFRESH_STATUS, "N").equalsIgnoreCase("Y")) {
out.println("<META http-equiv=\"Refresh\" content=\"10;url=" + convertContextPath(CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">");
}
out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
out.println("</HEAD>");
out.println("<BODY>");
out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "TransStatusServlet.TopTransStatus", transName)) + "</H1>");
try {
out.println("<table border=\"1\">");
out.print("<tr> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.TransName") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.CarteObjectId") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.TransStatus") + "</th> </tr>");
out.print("<tr>");
out.print("<td>" + Encode.forHtml(transName) + "</td>");
out.print("<td>" + Encode.forHtml(id) + "</td>");
out.print("<td>" + Encode.forHtml(trans.getStatus()) + "</td>");
out.print("</tr>");
out.print("</table>");
out.print("<p>");
// Get the transformation image
//
// out.print("<a href=\"" + convertContextPath(GetTransImageServlet.CONTEXT_PATH) + "?name=" +
// URLEncoder.encode(transName, "UTF-8") + "&id="+id+"\">"
// + BaseMessages.getString(PKG, "TransStatusServlet.GetTransImage") + "</a>");
Point max = trans.getTransMeta().getMaximum();
max.x += 20;
max.y += 20;
out.print("<iframe height=\"" + max.y + "\" width=\"" + max.x + "\" seamless src=\"" + convertContextPath(GetTransImageServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\"></iframe>");
out.print("<p>");
if ((trans.isFinished() && trans.isRunning()) || (!trans.isRunning() && !trans.isPreparing() && !trans.isInitializing())) {
out.print("<a href=\"" + convertContextPath(StartTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.StartTrans") + "</a>");
out.print("<p>");
out.print("<a href=\"" + convertContextPath(PrepareExecutionTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.PrepareTrans") + "</a><br>");
} else if (trans.isRunning()) {
out.print("<a href=\"" + convertContextPath(PauseTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "PauseStatusServlet.PauseResumeTrans") + "</a><br>");
out.print("<a href=\"" + convertContextPath(StopTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.StopTrans") + "</a><br>");
out.print("<a href=\"" + convertContextPath(StopTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "&inputOnly=Y" + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.SafeStopTrans") + "</a>");
out.print("<p>");
}
out.print("<a href=\"" + convertContextPath(CleanupTransServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.CleanupTrans") + "</a>");
out.print("<p>");
out.println("<table border=\"1\">");
out.print("<tr> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Stepname") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.CopyNr") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Read") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Written") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Input") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Output") + "</th> " + "<th>" + BaseMessages.getString(PKG, "TransStatusServlet.Updated") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Rejected") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Errors") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Active") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.Time") + "</th> " + "<th>" + BaseMessages.getString(PKG, "TransStatusServlet.Speed") + "</th> <th>" + BaseMessages.getString(PKG, "TransStatusServlet.prinout") + "</th> </tr>");
for (int i = 0; i < trans.nrSteps(); i++) {
StepInterface step = trans.getRunThread(i);
if ((step.isRunning()) || step.getStatus() != StepExecutionStatus.STATUS_EMPTY) {
StepStatus stepStatus = new StepStatus(step);
boolean snif = false;
if (step.isRunning() && !step.isStopped() && !step.isPaused()) {
snif = true;
String sniffLink = " <a href=\"" + convertContextPath(SniffStepServlet.CONTEXT_PATH) + "?trans=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "&lines=50" + "©nr=" + step.getCopy() + "&type=" + SniffStepServlet.TYPE_OUTPUT + "&step=" + URLEncoder.encode(step.getStepname(), "UTF-8") + "\">" + Encode.forHtml(stepStatus.getStepname()) + "</a>";
stepStatus.setStepname(sniffLink);
}
out.print(stepStatus.getHTMLTableRow(snif));
}
}
out.println("</table>");
out.println("<p>");
out.print("<a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "&xml=y\">" + BaseMessages.getString(PKG, "TransStatusServlet.ShowAsXml") + "</a><br>");
out.print("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><br>");
out.print("<p><a href=\"" + convertContextPath(GetTransStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.Refresh") + "</a>");
// Put the logging below that.
out.println("<p>");
out.println("<textarea id=\"translog\" cols=\"120\" rows=\"20\" " + "wrap=\"off\" name=\"Transformation log\" readonly=\"readonly\">" + Encode.forHtml(getLogText(trans, startLineNr, lastLineNr)) + "</textarea>");
out.println("<script type=\"text/javascript\"> ");
out.println(" translog.scrollTop=translog.scrollHeight; ");
out.println("</script> ");
out.println("<p>");
} catch (Exception ex) {
out.println("<p>");
out.println("<pre>");
out.println(Encode.forHtml(Const.getStackTracker(ex)));
out.println("</pre>");
}
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
} else {
PrintWriter out = response.getWriter();
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "TransStatusServlet.Log.CoundNotFindSpecTrans", transName)));
} else {
out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "TransStatusServlet.Log.CoundNotFindTrans", transName)) + "</H1>");
out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>");
}
}
}
use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.
the class TransformationResource method getTransformationStatus.
@GET
@Path("/status/{id : .+}")
@Produces({ MediaType.APPLICATION_JSON })
public TransformationStatus getTransformationStatus(@PathParam("id") String id) {
TransformationStatus status = new TransformationStatus();
// find trans
Trans trans = CarteResource.getTransformation(id);
CarteObjectEntry entry = CarteResource.getCarteObjectEntry(id);
status.setId(entry.getId());
status.setName(entry.getName());
status.setStatus(trans.getStatus());
for (int i = 0; i < trans.nrSteps(); i++) {
StepInterface step = trans.getRunThread(i);
if ((step.isRunning()) || step.getStatus() != StepExecutionStatus.STATUS_EMPTY) {
StepStatus stepStatus = new StepStatus(step);
status.addStepStatus(stepStatus);
}
}
return status;
}
Aggregations