use of org.pentaho.di.trans.step.StepErrorMeta in project data-access by pentaho.
the class StagingTransformGenerator method prepareTransform.
protected void prepareTransform(Trans trans, final IPentahoSession session) throws KettleException {
trans.prepareExecution(trans.getArguments());
StepInterface tableOutputStep = trans.findRunThread(TABLE_OUTPUT);
if (tableOutputStep != null) {
StepErrorMeta tableOutputErrorMeta = new StepErrorMeta(trans.getTransMeta(), tableOutputStep.getStepMeta()) {
public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
// don't overwhelm the user with too many errors
if (errorRowCount < maxErrorRows) {
StringBuffer sb = new StringBuffer();
sb.append("Rejected Row: ");
for (Object rowData : row) {
sb.append(rowData);
sb.append(", ");
}
sb.append("\r\n");
if (transformStats != null) {
transformStats.getErrors().add(sb.toString() + errorDescriptions);
}
}
errorRowCount++;
transformStats.setErrorCount(errorRowCount);
super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
}
};
StepMeta outputDummyStepMeta = addDummyStep(trans.getTransMeta(), "TableOutputErrorDummy");
tableOutputErrorMeta.setTargetStep(outputDummyStepMeta);
tableOutputErrorMeta.setEnabled(true);
tableOutputStep.getStepMeta().setStepErrorMeta(tableOutputErrorMeta);
}
}
use of org.pentaho.di.trans.step.StepErrorMeta in project pentaho-kettle by pentaho.
the class Spoon method pasteXML.
public void pasteXML(TransMeta transMeta, String clipcontent, Point loc) {
if (RepositorySecurityUI.verifyOperations(shell, rep, RepositoryOperation.MODIFY_TRANSFORMATION, RepositoryOperation.EXECUTE_TRANSFORMATION)) {
return;
}
try {
Document doc = XMLHandler.loadXMLString(clipcontent);
Node transNode = XMLHandler.getSubNode(doc, Spoon.XML_TAG_TRANSFORMATION_STEPS);
// De-select all, re-select pasted steps...
transMeta.unselectAll();
Node stepsNode = XMLHandler.getSubNode(transNode, "steps");
int nr = XMLHandler.countNodes(stepsNode, "step");
if (getLog().isDebug()) {
// "I found "+nr+" steps to paste on location: "
getLog().logDebug(BaseMessages.getString(PKG, "Spoon.Log.FoundSteps", "" + nr) + loc);
}
StepMeta[] steps = new StepMeta[nr];
ArrayList<String> stepOldNames = new ArrayList<>(nr);
// Point min = new Point(loc.x, loc.y);
Point min = new Point(99999999, 99999999);
// Load the steps...
for (int i = 0; i < nr; i++) {
Node stepNode = XMLHandler.getSubNodeByNr(stepsNode, "step", i);
steps[i] = new StepMeta(stepNode, transMeta.getDatabases(), metaStore);
if (loc != null) {
Point p = steps[i].getLocation();
if (min.x > p.x) {
min.x = p.x;
}
if (min.y > p.y) {
min.y = p.y;
}
}
}
// Load the hops...
Node hopsNode = XMLHandler.getSubNode(transNode, "order");
nr = XMLHandler.countNodes(hopsNode, "hop");
if (getLog().isDebug()) {
// "I found "+nr+" hops to paste."
getLog().logDebug(BaseMessages.getString(PKG, "Spoon.Log.FoundHops", "" + nr));
}
TransHopMeta[] hops = new TransHopMeta[nr];
for (int i = 0; i < nr; i++) {
Node hopNode = XMLHandler.getSubNodeByNr(hopsNode, "hop", i);
hops[i] = new TransHopMeta(hopNode, Arrays.asList(steps));
}
// This is the offset:
Point offset = new Point(loc.x - min.x, loc.y - min.y);
// Undo/redo object positions...
int[] position = new int[steps.length];
for (int i = 0; i < steps.length; i++) {
Point p = steps[i].getLocation();
String name = steps[i].getName();
steps[i].setLocation(p.x + offset.x, p.y + offset.y);
steps[i].setDraw(true);
// Check the name, find alternative...
stepOldNames.add(name);
steps[i].setName(transMeta.getAlternativeStepname(name));
transMeta.addStep(steps[i]);
position[i] = transMeta.indexOfStep(steps[i]);
steps[i].setSelected(true);
}
// Add the hops too...
for (TransHopMeta hop : hops) {
transMeta.addTransHop(hop);
}
// Load the notes...
Node notesNode = XMLHandler.getSubNode(transNode, "notepads");
nr = XMLHandler.countNodes(notesNode, "notepad");
if (getLog().isDebug()) {
// "I found "+nr+" notepads to paste."
getLog().logDebug(BaseMessages.getString(PKG, "Spoon.Log.FoundNotepads", "" + nr));
}
NotePadMeta[] notes = new NotePadMeta[nr];
for (int i = 0; i < notes.length; i++) {
Node noteNode = XMLHandler.getSubNodeByNr(notesNode, "notepad", i);
notes[i] = new NotePadMeta(noteNode);
Point p = notes[i].getLocation();
notes[i].setLocation(p.x + offset.x, p.y + offset.y);
transMeta.addNote(notes[i]);
notes[i].setSelected(true);
}
// Set the source and target steps ...
for (StepMeta step : steps) {
StepMetaInterface smi = step.getStepMetaInterface();
smi.searchInfoAndTargetSteps(transMeta.getSteps());
}
// Set the error handling hops
Node errorHandlingNode = XMLHandler.getSubNode(transNode, TransMeta.XML_TAG_STEP_ERROR_HANDLING);
int nrErrorHandlers = XMLHandler.countNodes(errorHandlingNode, StepErrorMeta.XML_ERROR_TAG);
for (int i = 0; i < nrErrorHandlers; i++) {
Node stepErrorMetaNode = XMLHandler.getSubNodeByNr(errorHandlingNode, StepErrorMeta.XML_ERROR_TAG, i);
StepErrorMeta stepErrorMeta = new StepErrorMeta(transMeta.getParentVariableSpace(), stepErrorMetaNode, transMeta.getSteps());
// Handle pasting multiple times, need to update source and target step names
int srcStepPos = stepOldNames.indexOf(stepErrorMeta.getSourceStep().getName());
int tgtStepPos = stepOldNames.indexOf(stepErrorMeta.getTargetStep().getName());
StepMeta sourceStep = transMeta.findStep(steps[srcStepPos].getName());
if (sourceStep != null) {
sourceStep.setStepErrorMeta(stepErrorMeta);
}
sourceStep.setStepErrorMeta(null);
if (tgtStepPos >= 0) {
sourceStep.setStepErrorMeta(stepErrorMeta);
StepMeta targetStep = transMeta.findStep(steps[tgtStepPos].getName());
stepErrorMeta.setSourceStep(sourceStep);
stepErrorMeta.setTargetStep(targetStep);
}
}
// Save undo information too...
addUndoNew(transMeta, steps, position, false);
int[] hopPos = new int[hops.length];
for (int i = 0; i < hops.length; i++) {
hopPos[i] = transMeta.indexOfTransHop(hops[i]);
}
addUndoNew(transMeta, hops, hopPos, true);
int[] notePos = new int[notes.length];
for (int i = 0; i < notes.length; i++) {
notePos[i] = transMeta.indexOfNote(notes[i]);
}
addUndoNew(transMeta, notes, notePos, true);
if (transMeta.haveStepsChanged()) {
refreshTree();
refreshGraph();
}
} catch (KettleException e) {
// "Error pasting steps...",
// "I was unable to paste steps to this transformation"
new ErrorDialog(shell, BaseMessages.getString(PKG, "Spoon.Dialog.UnablePasteSteps.Title"), BaseMessages.getString(PKG, "Spoon.Dialog.UnablePasteSteps.Message"), e);
}
}
use of org.pentaho.di.trans.step.StepErrorMeta in project pentaho-kettle by pentaho.
the class SynchronizeAfterMergeIT method setUp.
@Before
public void setUp() throws KettleDatabaseException, SQLException {
connection = DriverManager.getConnection("jdbc:h2:mem:PERSON;");
connection.setAutoCommit(false);
PreparedStatement stmt = connection.prepareStatement("CREATE TABLE PERSON (ID INT PRIMARY KEY, personName VARCHAR(64) )");
stmt.execute();
stmt.close();
stmt = connection.prepareStatement("INSERT INTO PERSON (ID, personName) VALUES (?, ?)");
for (int i = 0; i < ROW_FOR_UPDATE + ROW_FOR_DELETE; i++) {
stmt.setInt(1, i);
stmt.setString(2, "personName" + i);
stmt.addBatch();
}
stmt.executeBatch();
stmt.close();
connection.commit();
PluginRegistry pluginRegistry = PluginRegistry.getInstance();
transMeta = new TransMeta();
transMeta.setName("SynchronizeAfterMerge");
InjectorMeta injectorMeta = new InjectorMeta();
String injectorPid = pluginRegistry.getPluginId(StepPluginType.class, injectorMeta);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, injectorMeta);
transMeta.addStep(injectorStep);
DatabaseMeta dbMeta = spy(new DatabaseMeta());
dbMeta.setDatabaseType("H2");
when(dbMeta.getURL()).thenReturn("jdbc:h2:mem:PERSON;");
when(dbMeta.supportsErrorHandlingOnBatchUpdates()).thenReturn(false);
SynchronizeAfterMergeMeta synchronizeAfterMergeMeta = new SynchronizeAfterMergeMeta();
// set commit size
synchronizeAfterMergeMeta.setCommitSize(COMMIT_SIZE);
synchronizeAfterMergeMeta.setDatabaseMeta(dbMeta);
synchronizeAfterMergeMeta.setKeyCondition(new String[] { "=" });
synchronizeAfterMergeMeta.setKeyLookup(new String[] { "ID" });
synchronizeAfterMergeMeta.setKeyStream(new String[] { "personName" });
synchronizeAfterMergeMeta.setKeyStream2(new String[] { null });
synchronizeAfterMergeMeta.setUpdate(new Boolean[] { Boolean.TRUE });
synchronizeAfterMergeMeta.setOperationOrderField("flag");
synchronizeAfterMergeMeta.setOrderDelete(DELETE_FLAG);
synchronizeAfterMergeMeta.setOrderInsert(INSERT_FLAG);
synchronizeAfterMergeMeta.setOrderUpdate(UPDATE_FLAG);
synchronizeAfterMergeMeta.setPerformLookup(true);
synchronizeAfterMergeMeta.setTableName("PERSON");
synchronizeAfterMergeMeta.settablenameInField(false);
synchronizeAfterMergeMeta.settablenameField(null);
synchronizeAfterMergeMeta.setUseBatchUpdate(true);
synchronizeAfterMergeMeta.setUpdateLookup(new String[] { "ID" });
synchronizeAfterMergeMeta.setUpdateStream(new String[] { "personName" });
String synchronizeAfterMergePid = pluginRegistry.getPluginId(StepPluginType.class, synchronizeAfterMergeStepname);
StepMeta synchronizeAfterMerge = new StepMeta(synchronizeAfterMergePid, synchronizeAfterMergeStepname, synchronizeAfterMergeMeta);
transMeta.addStep(synchronizeAfterMerge);
String dummyResultStepName = "dummyResultStepName";
DummyTransMeta dummyResultTransMeta = new DummyTransMeta();
String dummyResultPid = pluginRegistry.getPluginId(StepPluginType.class, dummyResultTransMeta);
StepMeta dummyResultStep = new StepMeta(dummyResultPid, dummyResultStepName, dummyResultTransMeta);
transMeta.addStep(dummyResultStep);
String dummyErrorStepName = "dummyErrorStepName";
DummyTransMeta dummyErrorTransMeta = new DummyTransMeta();
String dummyErrorPid = pluginRegistry.getPluginId(StepPluginType.class, dummyErrorTransMeta);
StepMeta dummyErrorStep = new StepMeta(dummyErrorPid, dummyErrorStepName, dummyErrorTransMeta);
transMeta.addStep(dummyErrorStep);
StepErrorMeta stepErrorMeta = new StepErrorMeta(transMeta, synchronizeAfterMerge, dummyErrorStep);
stepErrorMeta.setEnabled(true);
synchronizeAfterMerge.setStepErrorMeta(stepErrorMeta);
TransHopMeta injSynch = new TransHopMeta(injectorStep, synchronizeAfterMerge);
transMeta.addTransHop(injSynch);
TransHopMeta synchDummyResult = new TransHopMeta(synchronizeAfterMerge, dummyResultStep);
transMeta.addTransHop(synchDummyResult);
TransHopMeta synchDummyError = new TransHopMeta(synchronizeAfterMerge, dummyErrorStep);
transMeta.addTransHop(synchDummyError);
}
use of org.pentaho.di.trans.step.StepErrorMeta in project pentaho-kettle by pentaho.
the class KettleDatabaseRepositoryStepDelegate method loadStepErrorMeta.
public StepErrorMeta loadStepErrorMeta(VariableSpace variables, StepMeta stepMeta, List<StepMeta> steps) throws KettleException {
StepErrorMeta meta = new StepErrorMeta(variables, stepMeta);
meta.setTargetStep(StepMeta.findStep(steps, repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_target_step")));
meta.setEnabled(repository.getStepAttributeBoolean(stepMeta.getObjectId(), "step_error_handling_is_enabled"));
meta.setNrErrorsValuename(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_nr_valuename"));
meta.setErrorDescriptionsValuename(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_descriptions_valuename"));
meta.setErrorFieldsValuename(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_fields_valuename"));
meta.setErrorCodesValuename(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_codes_valuename"));
meta.setMaxErrors(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_max_errors"));
meta.setMaxPercentErrors(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_max_pct_errors"));
meta.setMinPercentRows(repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_min_pct_rows"));
return meta;
}
use of org.pentaho.di.trans.step.StepErrorMeta in project pentaho-kettle by pentaho.
the class KettleDatabaseRepositoryTransDelegate method loadTransformation.
/**
* Read a transformation with a certain name from a repository
*
* @param transname
* The name of the transformation.
* @param repdir
* the path to the repository directory
* @param monitor
* The progress monitor to display the progress of the file-open operation in a dialog
* @param setInternalVariables
* true if you want to set the internal variables based on this transformation information
*/
public TransMeta loadTransformation(TransMeta transMeta, String transname, RepositoryDirectoryInterface repdir, ProgressMonitorListener monitor, boolean setInternalVariables) throws KettleException {
transMeta.setRepository(repository);
transMeta.setMetaStore(repository.metaStore);
synchronized (repository) {
try {
String pathAndName = repdir.isRoot() ? repdir + transname : repdir + RepositoryDirectory.DIRECTORY_SEPARATOR + transname;
transMeta.setName(transname);
transMeta.setRepositoryDirectory(repdir);
// Get the transformation id
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LookingForTransformation", transname, repdir.getPath()));
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTransformationInfoTask.Title"));
}
transMeta.setObjectId(getTransformationID(transname, repdir.getObjectId()));
if (monitor != null) {
monitor.worked(1);
}
// If no valid id is available in the database, then give error...
if (transMeta.getObjectId() != null) {
ObjectId[] noteids = repository.getTransNoteIDs(transMeta.getObjectId());
ObjectId[] stepids = repository.getStepIDs(transMeta.getObjectId());
ObjectId[] hopids = getTransHopIDs(transMeta.getObjectId());
int nrWork = 3 + noteids.length + stepids.length + hopids.length;
if (monitor != null) {
monitor.beginTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationTask.Title") + pathAndName, nrWork);
}
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingTransformation", transMeta.getName()));
}
// Load the common database connections
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheAvailableSharedObjectsTask.Title"));
}
try {
transMeta.setSharedObjects(readTransSharedObjects(transMeta));
} catch (Exception e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.ErrorReadingSharedObjects.Message", e.toString()));
log.logError(Const.getStackTracker(e));
}
if (monitor != null) {
monitor.worked(1);
}
// Load the notes...
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingNoteTask.Title"));
}
for (int i = 0; i < noteids.length; i++) {
NotePadMeta ni = repository.notePadDelegate.loadNotePadMeta(noteids[i]);
if (transMeta.indexOfNote(ni) < 0) {
transMeta.addNote(ni);
}
if (monitor != null) {
monitor.worked(1);
}
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepsTask.Title"));
}
// read all the attributes
repository.connectionDelegate.fillStepAttributesBuffer(transMeta.getObjectId());
// on one go!
for (int i = 0; i < stepids.length; i++) {
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingStepWithID") + stepids[i]);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepTask.Title") + (i + 1) + "/" + (stepids.length));
}
StepMeta stepMeta = repository.stepDelegate.loadStepMeta(stepids[i], transMeta.getDatabases(), transMeta.getPartitionSchemas());
if (stepMeta.isMissing()) {
transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
}
// In this case, we just add or replace the shared steps.
// The repository is considered "more central"
transMeta.addOrReplaceStep(stepMeta);
if (monitor != null) {
monitor.worked(1);
}
}
if (monitor != null) {
monitor.worked(1);
}
// clear the buffer (should be empty anyway)
repository.connectionDelegate.setStepAttributesBuffer(null);
// Have all StreamValueLookups, etc. reference the correct source steps...
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMetaInterface sii = transMeta.getStep(i).getStepMetaInterface();
sii.searchInfoAndTargetSteps(transMeta.getSteps());
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationDetailsTask.Title"));
}
loadRepTrans(transMeta);
if (monitor != null) {
monitor.worked(1);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingHopTask.Title"));
}
for (int i = 0; i < hopids.length; i++) {
TransHopMeta hi = loadTransHopMeta(hopids[i], transMeta.getSteps());
if (hi != null) {
transMeta.addTransHop(hi);
}
if (monitor != null) {
monitor.worked(1);
}
}
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepPartitioningMeta stepPartitioningMeta = transMeta.getStep(i).getStepPartitioningMeta();
if (stepPartitioningMeta != null) {
stepPartitioningMeta.setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
}
}
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas());
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheDependenciesTask.Title"));
}
ObjectId[] depids = repository.getTransDependencyIDs(transMeta.getObjectId());
for (int i = 0; i < depids.length; i++) {
TransDependency td = loadTransDependency(depids[i], transMeta.getDatabases());
transMeta.addDependency(td);
}
if (monitor != null) {
monitor.worked(1);
}
// Load the group attributes map
//
transMeta.setAttributesMap(loadTransAttributesMap(transMeta.getObjectId()));
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMeta stepMeta = transMeta.getStep(i);
String sourceStep = repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_source_step");
if (sourceStep != null) {
StepErrorMeta stepErrorMeta = repository.stepDelegate.loadStepErrorMeta(transMeta, stepMeta, transMeta.getSteps());
// a bit of a trick, I know.
stepErrorMeta.getSourceStep().setStepErrorMeta(stepErrorMeta);
}
}
// Load all the log tables for the transformation...
//
RepositoryAttributeInterface attributeInterface = new KettleDatabaseRepositoryTransAttribute(repository.connectionDelegate, transMeta.getObjectId());
for (LogTableInterface logTable : transMeta.getLogTables()) {
logTable.loadFromRepository(attributeInterface);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SortingStepsTask.Title"));
}
transMeta.sortSteps();
if (monitor != null) {
monitor.worked(1);
}
if (monitor != null) {
monitor.done();
}
} else {
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.TransformationDoesNotExist") + transMeta.getName());
}
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation2", transname, String.valueOf(transMeta.getRepositoryDirectory() == null)));
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation", transname, transMeta.getRepositoryDirectory().getPath()));
}
// close prepared statements, minimize locking etc.
//
repository.connectionDelegate.closeAttributeLookupPreparedStatements();
return transMeta;
} catch (KettleDatabaseException e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation"), e);
} catch (Exception e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation2"), e);
} finally {
transMeta.initializeVariablesFrom(null);
if (setInternalVariables) {
transMeta.setInternalKettleVariables();
}
}
}
}
Aggregations