use of org.pentaho.di.trans.steps.mappingoutput.MappingOutput in project pentaho-kettle by pentaho.
the class BaseStep method findInputRowSet.
/**
* Find input row set.
*
* @param from the from
* @param fromcopy the fromcopy
* @param to the to
* @param tocopy the tocopy
* @return the row set
*/
public RowSet findInputRowSet(String from, int fromcopy, String to, int tocopy) {
synchronized (inputRowSetsLock) {
for (RowSet rs : inputRowSets) {
if (rs.getOriginStepName().equalsIgnoreCase(from) && rs.getDestinationStepName().equalsIgnoreCase(to) && rs.getOriginStepCopy() == fromcopy && rs.getDestinationStepCopy() == tocopy) {
return rs;
}
}
}
// See if the rowset is part of the output of a mapping source step...
//
// Lookup step "From"
//
StepMeta mappingStep = transMeta.findStep(from);
//
if (mappingStep != null && mappingStep.isMapping()) {
// In this case we can cast the step thread to a Mapping...
//
List<StepInterface> baseSteps = trans.findBaseSteps(from);
if (baseSteps.size() == 1) {
Mapping mapping = (Mapping) baseSteps.get(0);
// Find the appropriate rowset in the mapping...
// The rowset in question has been passed over to a Mapping Input step inside the Mapping transformation.
//
MappingOutput[] outputs = mapping.getMappingTrans().findMappingOutput();
for (MappingOutput output : outputs) {
for (RowSet rs : output.getOutputRowSets()) {
//
if (rs.getDestinationStepName().equalsIgnoreCase(to)) {
return rs;
}
}
}
}
}
return null;
}
use of org.pentaho.di.trans.steps.mappingoutput.MappingOutput in project pentaho-kettle by pentaho.
the class SimpleMapping method prepareMappingExecution.
public void prepareMappingExecution() throws KettleException {
SimpleMappingData simpleMappingData = getData();
// Create the transformation from meta-data...
simpleMappingData.mappingTrans = new Trans(simpleMappingData.mappingTransMeta, this);
// Set the parameters values in the mapping.
//
StepWithMappingMeta.activateParams(simpleMappingData.mappingTrans, simpleMappingData.mappingTrans, this, simpleMappingData.mappingTransMeta.listParameters(), meta.getMappingParameters().getVariable(), meta.getMappingParameters().getInputField());
if (simpleMappingData.mappingTransMeta.getTransformationType() != TransformationType.Normal) {
simpleMappingData.mappingTrans.getTransMeta().setUsingThreadPriorityManagment(false);
}
// Leave a path up so that we can set variables in sub-transformations...
//
simpleMappingData.mappingTrans.setParentTrans(getTrans());
// Pass down the safe mode flag to the mapping...
//
simpleMappingData.mappingTrans.setSafeModeEnabled(getTrans().isSafeModeEnabled());
// Pass down the metrics gathering flag:
//
simpleMappingData.mappingTrans.setGatheringMetrics(getTrans().isGatheringMetrics());
// Also set the name of this step in the mapping transformation for logging
// purposes
//
simpleMappingData.mappingTrans.setMappingStepName(getStepname());
initServletConfig();
//
try {
simpleMappingData.mappingTrans.prepareExecution(getTrans().getArguments());
} catch (KettleException e) {
throw new KettleException(BaseMessages.getString(PKG, "SimpleMapping.Exception.UnableToPrepareExecutionOfMapping"), e);
}
// If there is no read/write logging step set, we can insert the data from
// the first mapping input/output step...
//
MappingInput[] mappingInputs = simpleMappingData.mappingTrans.findMappingInput();
if (mappingInputs.length == 0) {
throw new KettleException("The simple mapping step needs one Mapping Input step to write to in the sub-transformation");
}
if (mappingInputs.length > 1) {
throw new KettleException("The simple mapping step does not support multiple Mapping Input steps to write to in the sub-transformation");
}
simpleMappingData.mappingInput = mappingInputs[0];
simpleMappingData.mappingInput.setConnectorSteps(new StepInterface[0], new ArrayList<MappingValueRename>(), null);
// LogTableField readField = data.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
// if (readField.getSubject() == null) {
// readField.setSubject(data.mappingInput.getStepMeta());
// }
MappingOutput[] mappingOutputs = simpleMappingData.mappingTrans.findMappingOutput();
if (mappingOutputs.length == 0) {
throw new KettleException("The simple mapping step needs one Mapping Output step to read from in the sub-transformation");
}
if (mappingOutputs.length > 1) {
throw new KettleException("The simple mapping step does not support " + "multiple Mapping Output steps to read from in the sub-transformation");
}
simpleMappingData.mappingOutput = mappingOutputs[0];
// LogTableField writeField = data.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
// if (writeField.getSubject() == null && data.mappingOutputs != null && data.mappingOutputs.length >= 1) {
// writeField.setSubject(data.mappingOutputs[0].getStepMeta());
// }
// Finally, add the mapping transformation to the active sub-transformations
// map in the parent transformation
//
getTrans().addActiveSubTransformation(getStepname(), simpleMappingData.mappingTrans);
}
use of org.pentaho.di.trans.steps.mappingoutput.MappingOutput in project pentaho-kettle by pentaho.
the class Mapping method processRow.
/**
* Process a single row. In our case, we send one row of data to a piece of transformation. In the transformation, we
* look up the MappingInput step to send our rows to it. As a consequence, for the time being, there can only be one
* MappingInput and one MappingOutput step in the Mapping.
*/
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
try {
meta = (MappingMeta) smi;
setData((MappingData) sdi);
MappingInput[] mappingInputs = getData().getMappingTrans().findMappingInput();
MappingOutput[] mappingOutputs = getData().getMappingTrans().findMappingOutput();
getData().wasStarted = true;
switch(getData().mappingTransMeta.getTransformationType()) {
case Normal:
case SerialSingleThreaded:
// Before we start, let's see if there are loose ends to tie up...
//
List<RowSet> inputRowSets = getInputRowSets();
if (!inputRowSets.isEmpty()) {
for (RowSet rowSet : inputRowSets) {
//
if (mappingInputs.length == 1) {
// Simple case: only one input mapping. Move the RowSet over
//
mappingInputs[0].addRowSetToInputRowSets(rowSet);
} else {
//
throw new KettleException("Unsupported situation detected where more than one Mapping Input step needs to be handled. " + "To solve it, insert a dummy step before the mapping step.");
}
}
clearInputRowSets();
}
//
if (!getRemoteInputSteps().isEmpty()) {
//
for (RemoteStep remoteStep : getRemoteInputSteps()) {
//
if (mappingInputs.length == 1) {
// Simple case: only one input mapping. Move the remote step over
//
mappingInputs[0].getRemoteInputSteps().add(remoteStep);
} else {
//
throw new KettleException("Unsupported situation detected where a remote input step is expecting data " + "to end up in a particular Mapping Input step of a sub-transformation. " + "To solve it, insert a dummy step before the mapping.");
}
}
getRemoteInputSteps().clear();
}
// Do the same thing for output row sets
//
List<RowSet> outputRowSets = getOutputRowSets();
if (!outputRowSets.isEmpty()) {
for (RowSet rowSet : outputRowSets) {
//
if (mappingOutputs.length == 1) {
// Simple case: only one output mapping. Move the RowSet over
//
mappingOutputs[0].addRowSetToOutputRowSets(rowSet);
} else {
//
throw new KettleException("Unsupported situation detected where more than one Mapping Output step needs to be handled. " + "To solve it, insert a dummy step after the mapping step.");
}
}
clearOutputRowSets();
}
//
if (!getRemoteOutputSteps().isEmpty()) {
//
for (RemoteStep remoteStep : getRemoteOutputSteps()) {
//
if (mappingOutputs.length == 1) {
// Simple case: only one output mapping. Move the remote step over
//
mappingOutputs[0].getRemoteOutputSteps().add(remoteStep);
} else {
//
throw new KettleException("Unsupported situation detected where a remote output step is expecting data " + "to end up in a particular Mapping Output step of a sub-transformation. " + "To solve it, insert a dummy step after the mapping.");
}
}
getRemoteOutputSteps().clear();
}
// Start the mapping/sub-transformation threads
//
getData().getMappingTrans().startThreads();
//
if (getTransMeta().getTransformationType() == TransformationType.Normal) {
getData().getMappingTrans().waitUntilFinished();
// Set some statistics from the mapping...
// This will show up in Spoon, etc.
//
Result result = getData().getMappingTrans().getResult();
setErrors(result.getNrErrors());
setLinesRead(result.getNrLinesRead());
setLinesWritten(result.getNrLinesWritten());
setLinesInput(result.getNrLinesInput());
setLinesOutput(result.getNrLinesOutput());
setLinesUpdated(result.getNrLinesUpdated());
setLinesRejected(result.getNrLinesRejected());
}
return false;
case SingleThreaded:
if (mappingInputs.length > 1 || mappingOutputs.length > 1) {
throw new KettleException("Multiple input or output steps are not supported for a single threaded mapping.");
}
if ((log != null) && log.isDebug()) {
List<RowSet> mappingInputRowSets = mappingInputs[0].getInputRowSets();
log.logDebug("# of input buffers: " + mappingInputRowSets.size());
if (mappingInputRowSets.size() > 0) {
log.logDebug("Input buffer 0 size: " + mappingInputRowSets.get(0).size());
}
}
// Now execute one batch...Basic logging
//
boolean result = getData().singleThreadedTransExcecutor.oneIteration();
if (!result) {
getData().singleThreadedTransExcecutor.dispose();
setOutputDone();
return false;
}
return true;
default:
throw new KettleException("Transformation type '" + getData().mappingTransMeta.getTransformationType().getDescription() + "' is an unsupported transformation type for a mapping");
}
} catch (Throwable t) {
//
if (getData().getMappingTrans() != null) {
getData().getMappingTrans().stopAll();
}
//
throw new KettleException(t);
}
}
use of org.pentaho.di.trans.steps.mappingoutput.MappingOutput in project pentaho-kettle by pentaho.
the class Mapping method prepareMappingExecution.
public void prepareMappingExecution() throws KettleException {
initTransFromMeta();
MappingData mappingData = getData();
//
try {
mappingData.getMappingTrans().prepareExecution(getTrans().getArguments());
} catch (KettleException e) {
throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToPrepareExecutionOfMapping"), e);
}
//
switch(mappingData.mappingTransMeta.getTransformationType()) {
case Normal:
case SerialSingleThreaded:
break;
case SingleThreaded:
mappingData.singleThreadedTransExcecutor = new SingleThreadedTransExecutor(mappingData.getMappingTrans());
if (!mappingData.singleThreadedTransExcecutor.init()) {
throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToInitSingleThreadedTransformation"));
}
break;
default:
break;
}
// If there is no read/write logging step set, we can insert the data from
// the first mapping input/output step...
//
MappingInput[] mappingInputs = mappingData.getMappingTrans().findMappingInput();
LogTableField readField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
if (readField.getSubject() == null && mappingInputs != null && mappingInputs.length >= 1) {
readField.setSubject(mappingInputs[0].getStepMeta());
}
MappingOutput[] mappingOutputs = mappingData.getMappingTrans().findMappingOutput();
LogTableField writeField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
if (writeField.getSubject() == null && mappingOutputs != null && mappingOutputs.length >= 1) {
writeField.setSubject(mappingOutputs[0].getStepMeta());
}
// Before we add rowsets and all, we should note that the mapping step did
// not receive ANY input and output rowsets.
// This is an exception to the general rule, built into
// Trans.prepareExecution()
//
// A Mapping Input step is supposed to read directly from the previous
// steps.
// A Mapping Output step is supposed to write directly to the next steps.
// OK, check the input mapping definitions and look up the steps to read
// from.
//
StepInterface[] sourceSteps;
for (MappingIODefinition inputDefinition : meta.getInputMappings()) {
//
if (!Utils.isEmpty(inputDefinition.getInputStepname())) {
StepInterface sourceStep = getTrans().findRunThread(inputDefinition.getInputStepname());
if (sourceStep == null) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getInputStepname()));
}
sourceSteps = new StepInterface[] { sourceStep };
} else {
// We have no defined source step.
// That means that we're reading from all input steps that this mapping
// step has.
//
List<StepMeta> prevSteps = getTransMeta().findPreviousSteps(getStepMeta());
// TODO: Handle remote steps from: getStepMeta().getRemoteInputSteps()
//
// Let's read data from all the previous steps we find...
// The origin is the previous step
// The target is the Mapping Input step.
//
sourceSteps = new StepInterface[prevSteps.size()];
for (int s = 0; s < sourceSteps.length; s++) {
sourceSteps[s] = getTrans().findRunThread(prevSteps.get(s).getName());
}
}
// What step are we writing to?
MappingInput mappingInputTarget = null;
MappingInput[] mappingInputSteps = mappingData.getMappingTrans().findMappingInput();
if (Utils.isEmpty(inputDefinition.getOutputStepname())) {
if (mappingInputSteps.length == 0) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingInputStepRequired"));
}
if (mappingInputSteps.length > 1) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingInputStepAllowed", "" + mappingInputSteps.length));
}
mappingInputTarget = mappingInputSteps[0];
} else {
// A target step was specified. See if we can find it...
for (int s = 0; s < mappingInputSteps.length && mappingInputTarget == null; s++) {
if (mappingInputSteps[s].getStepname().equals(inputDefinition.getOutputStepname())) {
mappingInputTarget = mappingInputSteps[s];
}
}
// If we still didn't find it it's a drag.
if (mappingInputTarget == null) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getOutputStepname()));
}
}
//
if (inputDefinition.isRenamingOnOutput()) {
addInputRenames(getData().inputRenameList, inputDefinition.getValueRenames());
}
mappingInputTarget.setConnectorSteps(sourceSteps, inputDefinition.getValueRenames(), getStepname());
}
//
for (MappingIODefinition outputDefinition : meta.getOutputMappings()) {
// OK, what is the source (input) step in the mapping: it's the mapping
// output step...
// What step are we reading from here?
//
MappingOutput mappingOutputSource = (MappingOutput) mappingData.getMappingTrans().findRunThread(outputDefinition.getInputStepname());
if (mappingOutputSource == null) {
// No source step was specified: we're reading from a single Mapping
// Output step.
// We should verify this if this is really the case...
//
MappingOutput[] mappingOutputSteps = mappingData.getMappingTrans().findMappingOutput();
if (mappingOutputSteps.length == 0) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingOutputStepRequired"));
}
if (mappingOutputSteps.length > 1) {
throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingOutputStepAllowed", "" + mappingOutputSteps.length));
}
mappingOutputSource = mappingOutputSteps[0];
}
// To what steps in this transformation are we writing to?
//
StepInterface[] targetSteps = pickupTargetStepsFor(outputDefinition);
// Now tell the mapping output step where to look...
// Also explain the mapping output steps how to rename the values back...
//
mappingOutputSource.setConnectorSteps(targetSteps, getData().inputRenameList, outputDefinition.getValueRenames());
// Is this mapping copying or distributing?
// Make sure the mapping output step mimics this behavior:
//
mappingOutputSource.setDistributed(isDistributed());
}
// Finally, add the mapping transformation to the active sub-transformations
// map in the parent transformation
//
getTrans().addActiveSubTransformation(getStepname(), getData().getMappingTrans());
}
use of org.pentaho.di.trans.steps.mappingoutput.MappingOutput in project pentaho-kettle by pentaho.
the class Trans method findMappingOutput.
/**
* Gets the mapping outputs for each step in the transformation.
*
* @return an array of MappingOutputs
*/
public MappingOutput[] findMappingOutput() {
List<MappingOutput> list = new ArrayList<>();
if (steps != null) {
// Look in threads and find the MappingInput step thread...
for (int i = 0; i < steps.size(); i++) {
StepMetaDataCombi smdc = steps.get(i);
StepInterface step = smdc.step;
if (step.getStepID().equalsIgnoreCase("MappingOutput")) {
list.add((MappingOutput) step);
}
}
}
return list.toArray(new MappingOutput[list.size()]);
}
Aggregations