Examples with MappingInput org.pentaho.di.trans.steps.mappinginput.MappingInput used on opensource projects

Example 1 with MappingInput

use of org.pentaho.di.trans.steps.mappinginput.MappingInput in project pentaho-kettle by pentaho.

the class SimpleMapping method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    SimpleMappingData simpleMappingData = getData();
    // Create the transformation from meta-data...
    simpleMappingData.mappingTrans = new Trans(simpleMappingData.mappingTransMeta, this);
    // Set the parameters values in the mapping.
    // 
    StepWithMappingMeta.activateParams(simpleMappingData.mappingTrans, simpleMappingData.mappingTrans, this, simpleMappingData.mappingTransMeta.listParameters(), meta.getMappingParameters().getVariable(), meta.getMappingParameters().getInputField());
    if (simpleMappingData.mappingTransMeta.getTransformationType() != TransformationType.Normal) {
        simpleMappingData.mappingTrans.getTransMeta().setUsingThreadPriorityManagment(false);
    }
    // Leave a path up so that we can set variables in sub-transformations...
    // 
    simpleMappingData.mappingTrans.setParentTrans(getTrans());
    // Pass down the safe mode flag to the mapping...
    // 
    simpleMappingData.mappingTrans.setSafeModeEnabled(getTrans().isSafeModeEnabled());
    // Pass down the metrics gathering flag:
    // 
    simpleMappingData.mappingTrans.setGatheringMetrics(getTrans().isGatheringMetrics());
    // Also set the name of this step in the mapping transformation for logging
    // purposes
    // 
    simpleMappingData.mappingTrans.setMappingStepName(getStepname());
    initServletConfig();
    // 
    try {
        simpleMappingData.mappingTrans.prepareExecution(getTrans().getArguments());
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "SimpleMapping.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // If there is no read/write logging step set, we can insert the data from
    // the first mapping input/output step...
    // 
    MappingInput[] mappingInputs = simpleMappingData.mappingTrans.findMappingInput();
    if (mappingInputs.length == 0) {
        throw new KettleException("The simple mapping step needs one Mapping Input step to write to in the sub-transformation");
    }
    if (mappingInputs.length > 1) {
        throw new KettleException("The simple mapping step does not support multiple Mapping Input steps to write to in the sub-transformation");
    }
    simpleMappingData.mappingInput = mappingInputs[0];
    simpleMappingData.mappingInput.setConnectorSteps(new StepInterface[0], new ArrayList<MappingValueRename>(), null);
    // LogTableField readField = data.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
    // if (readField.getSubject() == null) {
    // readField.setSubject(data.mappingInput.getStepMeta());
    // }
    MappingOutput[] mappingOutputs = simpleMappingData.mappingTrans.findMappingOutput();
    if (mappingOutputs.length == 0) {
        throw new KettleException("The simple mapping step needs one Mapping Output step to read from in the sub-transformation");
    }
    if (mappingOutputs.length > 1) {
        throw new KettleException("The simple mapping step does not support " + "multiple Mapping Output steps to read from in the sub-transformation");
    }
    simpleMappingData.mappingOutput = mappingOutputs[0];
    // LogTableField writeField = data.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
    // if (writeField.getSubject() == null && data.mappingOutputs != null && data.mappingOutputs.length >= 1) {
    // writeField.setSubject(data.mappingOutputs[0].getStepMeta());
    // }
    // Finally, add the mapping transformation to the active sub-transformations
    // map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), simpleMappingData.mappingTrans);
}

Example 2 with MappingInput

use of org.pentaho.di.trans.steps.mappinginput.MappingInput in project pentaho-kettle by pentaho.

the class Mapping method processRow.

/**
 * Process a single row. In our case, we send one row of data to a piece of transformation. In the transformation, we
 * look up the MappingInput step to send our rows to it. As a consequence, for the time being, there can only be one
 * MappingInput and one MappingOutput step in the Mapping.
 */
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    try {
        meta = (MappingMeta) smi;
        setData((MappingData) sdi);
        MappingInput[] mappingInputs = getData().getMappingTrans().findMappingInput();
        MappingOutput[] mappingOutputs = getData().getMappingTrans().findMappingOutput();
        getData().wasStarted = true;
        switch(getData().mappingTransMeta.getTransformationType()) {
            case Normal:
            case SerialSingleThreaded:
                // Before we start, let's see if there are loose ends to tie up...
                // 
                List<RowSet> inputRowSets = getInputRowSets();
                if (!inputRowSets.isEmpty()) {
                    for (RowSet rowSet : inputRowSets) {
                        // 
                        if (mappingInputs.length == 1) {
                            // Simple case: only one input mapping. Move the RowSet over
                            // 
                            mappingInputs[0].addRowSetToInputRowSets(rowSet);
                        } else {
                            // 
                            throw new KettleException("Unsupported situation detected where more than one Mapping Input step needs to be handled.  " + "To solve it, insert a dummy step before the mapping step.");
                        }
                    }
                    clearInputRowSets();
                }
                // 
                if (!getRemoteInputSteps().isEmpty()) {
                    // 
                    for (RemoteStep remoteStep : getRemoteInputSteps()) {
                        // 
                        if (mappingInputs.length == 1) {
                            // Simple case: only one input mapping. Move the remote step over
                            // 
                            mappingInputs[0].getRemoteInputSteps().add(remoteStep);
                        } else {
                            // 
                            throw new KettleException("Unsupported situation detected where a remote input step is expecting data " + "to end up in a particular Mapping Input step of a sub-transformation.  " + "To solve it, insert a dummy step before the mapping.");
                        }
                    }
                    getRemoteInputSteps().clear();
                }
                // Do the same thing for output row sets
                // 
                List<RowSet> outputRowSets = getOutputRowSets();
                if (!outputRowSets.isEmpty()) {
                    for (RowSet rowSet : outputRowSets) {
                        // 
                        if (mappingOutputs.length == 1) {
                            // Simple case: only one output mapping. Move the RowSet over
                            // 
                            mappingOutputs[0].addRowSetToOutputRowSets(rowSet);
                        } else {
                            // 
                            throw new KettleException("Unsupported situation detected where more than one Mapping Output step needs to be handled.  " + "To solve it, insert a dummy step after the mapping step.");
                        }
                    }
                    clearOutputRowSets();
                }
                // 
                if (!getRemoteOutputSteps().isEmpty()) {
                    // 
                    for (RemoteStep remoteStep : getRemoteOutputSteps()) {
                        // 
                        if (mappingOutputs.length == 1) {
                            // Simple case: only one output mapping. Move the remote step over
                            // 
                            mappingOutputs[0].getRemoteOutputSteps().add(remoteStep);
                        } else {
                            // 
                            throw new KettleException("Unsupported situation detected where a remote output step is expecting data " + "to end up in a particular Mapping Output step of a sub-transformation.  " + "To solve it, insert a dummy step after the mapping.");
                        }
                    }
                    getRemoteOutputSteps().clear();
                }
                // Start the mapping/sub-transformation threads
                // 
                getData().getMappingTrans().startThreads();
                // 
                if (getTransMeta().getTransformationType() == TransformationType.Normal) {
                    getData().getMappingTrans().waitUntilFinished();
                    // Set some statistics from the mapping...
                    // This will show up in Spoon, etc.
                    // 
                    Result result = getData().getMappingTrans().getResult();
                    setErrors(result.getNrErrors());
                    setLinesRead(result.getNrLinesRead());
                    setLinesWritten(result.getNrLinesWritten());
                    setLinesInput(result.getNrLinesInput());
                    setLinesOutput(result.getNrLinesOutput());
                    setLinesUpdated(result.getNrLinesUpdated());
                    setLinesRejected(result.getNrLinesRejected());
                }
                return false;
            case SingleThreaded:
                if (mappingInputs.length > 1 || mappingOutputs.length > 1) {
                    throw new KettleException("Multiple input or output steps are not supported for a single threaded mapping.");
                }
                if ((log != null) && log.isDebug()) {
                    List<RowSet> mappingInputRowSets = mappingInputs[0].getInputRowSets();
                    log.logDebug("# of input buffers: " + mappingInputRowSets.size());
                    if (mappingInputRowSets.size() > 0) {
                        log.logDebug("Input buffer 0 size: " + mappingInputRowSets.get(0).size());
                    }
                }
                // Now execute one batch...Basic logging
                // 
                boolean result = getData().singleThreadedTransExcecutor.oneIteration();
                if (!result) {
                    getData().singleThreadedTransExcecutor.dispose();
                    setOutputDone();
                    return false;
                }
                return true;
            default:
                throw new KettleException("Transformation type '" + getData().mappingTransMeta.getTransformationType().getDescription() + "' is an unsupported transformation type for a mapping");
        }
    } catch (Throwable t) {
        // 
        if (getData().getMappingTrans() != null) {
            getData().getMappingTrans().stopAll();
        }
        // 
        throw new KettleException(t);
    }
}

Example 3 with MappingInput

use of org.pentaho.di.trans.steps.mappinginput.MappingInput in project pentaho-kettle by pentaho.

the class Mapping method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    initTransFromMeta();
    MappingData mappingData = getData();
    // 
    try {
        mappingData.getMappingTrans().prepareExecution(getTrans().getArguments());
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // 
    switch(mappingData.mappingTransMeta.getTransformationType()) {
        case Normal:
        case SerialSingleThreaded:
            break;
        case SingleThreaded:
            mappingData.singleThreadedTransExcecutor = new SingleThreadedTransExecutor(mappingData.getMappingTrans());
            if (!mappingData.singleThreadedTransExcecutor.init()) {
                throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToInitSingleThreadedTransformation"));
            }
            break;
        default:
            break;
    }
    // If there is no read/write logging step set, we can insert the data from
    // the first mapping input/output step...
    // 
    MappingInput[] mappingInputs = mappingData.getMappingTrans().findMappingInput();
    LogTableField readField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
    if (readField.getSubject() == null && mappingInputs != null && mappingInputs.length >= 1) {
        readField.setSubject(mappingInputs[0].getStepMeta());
    }
    MappingOutput[] mappingOutputs = mappingData.getMappingTrans().findMappingOutput();
    LogTableField writeField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
    if (writeField.getSubject() == null && mappingOutputs != null && mappingOutputs.length >= 1) {
        writeField.setSubject(mappingOutputs[0].getStepMeta());
    }
    // Before we add rowsets and all, we should note that the mapping step did
    // not receive ANY input and output rowsets.
    // This is an exception to the general rule, built into
    // Trans.prepareExecution()
    // 
    // A Mapping Input step is supposed to read directly from the previous
    // steps.
    // A Mapping Output step is supposed to write directly to the next steps.
    // OK, check the input mapping definitions and look up the steps to read
    // from.
    // 
    StepInterface[] sourceSteps;
    for (MappingIODefinition inputDefinition : meta.getInputMappings()) {
        // 
        if (!Utils.isEmpty(inputDefinition.getInputStepname())) {
            StepInterface sourceStep = getTrans().findRunThread(inputDefinition.getInputStepname());
            if (sourceStep == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getInputStepname()));
            }
            sourceSteps = new StepInterface[] { sourceStep };
        } else {
            // We have no defined source step.
            // That means that we're reading from all input steps that this mapping
            // step has.
            // 
            List<StepMeta> prevSteps = getTransMeta().findPreviousSteps(getStepMeta());
            // TODO: Handle remote steps from: getStepMeta().getRemoteInputSteps()
            // 
            // Let's read data from all the previous steps we find...
            // The origin is the previous step
            // The target is the Mapping Input step.
            // 
            sourceSteps = new StepInterface[prevSteps.size()];
            for (int s = 0; s < sourceSteps.length; s++) {
                sourceSteps[s] = getTrans().findRunThread(prevSteps.get(s).getName());
            }
        }
        // What step are we writing to?
        MappingInput mappingInputTarget = null;
        MappingInput[] mappingInputSteps = mappingData.getMappingTrans().findMappingInput();
        if (Utils.isEmpty(inputDefinition.getOutputStepname())) {
            if (mappingInputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingInputStepRequired"));
            }
            if (mappingInputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingInputStepAllowed", "" + mappingInputSteps.length));
            }
            mappingInputTarget = mappingInputSteps[0];
        } else {
            // A target step was specified. See if we can find it...
            for (int s = 0; s < mappingInputSteps.length && mappingInputTarget == null; s++) {
                if (mappingInputSteps[s].getStepname().equals(inputDefinition.getOutputStepname())) {
                    mappingInputTarget = mappingInputSteps[s];
                }
            }
            // If we still didn't find it it's a drag.
            if (mappingInputTarget == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getOutputStepname()));
            }
        }
        // 
        if (inputDefinition.isRenamingOnOutput()) {
            addInputRenames(getData().inputRenameList, inputDefinition.getValueRenames());
        }
        mappingInputTarget.setConnectorSteps(sourceSteps, inputDefinition.getValueRenames(), getStepname());
    }
    // 
    for (MappingIODefinition outputDefinition : meta.getOutputMappings()) {
        // OK, what is the source (input) step in the mapping: it's the mapping
        // output step...
        // What step are we reading from here?
        // 
        MappingOutput mappingOutputSource = (MappingOutput) mappingData.getMappingTrans().findRunThread(outputDefinition.getInputStepname());
        if (mappingOutputSource == null) {
            // No source step was specified: we're reading from a single Mapping
            // Output step.
            // We should verify this if this is really the case...
            // 
            MappingOutput[] mappingOutputSteps = mappingData.getMappingTrans().findMappingOutput();
            if (mappingOutputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingOutputStepRequired"));
            }
            if (mappingOutputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingOutputStepAllowed", "" + mappingOutputSteps.length));
            }
            mappingOutputSource = mappingOutputSteps[0];
        }
        // To what steps in this transformation are we writing to?
        // 
        StepInterface[] targetSteps = pickupTargetStepsFor(outputDefinition);
        // Now tell the mapping output step where to look...
        // Also explain the mapping output steps how to rename the values back...
        // 
        mappingOutputSource.setConnectorSteps(targetSteps, getData().inputRenameList, outputDefinition.getValueRenames());
        // Is this mapping copying or distributing?
        // Make sure the mapping output step mimics this behavior:
        // 
        mappingOutputSource.setDistributed(isDistributed());
    }
    // Finally, add the mapping transformation to the active sub-transformations
    // map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), getData().getMappingTrans());
}

Example 4 with MappingInput

use of org.pentaho.di.trans.steps.mappinginput.MappingInput in project pentaho-kettle by pentaho.

the class BaseStep method findOutputRowSet.

/**
 * Find an output rowset in a running transformation. It will also look at the "to" step to see if this is a mapping.
 * If it is, it will find the appropriate rowset in that transformation.
 *
 * @param from
 * @param fromcopy
 * @param to
 * @param tocopy
 * @return The rowset or null if none is found.
 */
public RowSet findOutputRowSet(String from, int fromcopy, String to, int tocopy) {
    synchronized (outputRowSetsLock) {
        for (RowSet rs : outputRowSets) {
            if (rs.getOriginStepName().equalsIgnoreCase(from) && rs.getDestinationStepName().equalsIgnoreCase(to) && rs.getOriginStepCopy() == fromcopy && rs.getDestinationStepCopy() == tocopy) {
                return rs;
            }
        }
    }
    // See if the rowset is part of the input of a mapping target step...
    // 
    // Lookup step "To"
    // 
    StepMeta mappingStep = transMeta.findStep(to);
    // 
    if (mappingStep != null && mappingStep.isMapping()) {
        // In this case we can cast the step thread to a Mapping...
        // 
        List<StepInterface> baseSteps = trans.findBaseSteps(to);
        if (baseSteps.size() == 1) {
            Mapping mapping = (Mapping) baseSteps.get(0);
            // Find the appropriate rowset in the mapping...
            // The rowset in question has been passed over to a Mapping Input step inside the Mapping transformation.
            // 
            MappingInput[] inputs = mapping.getMappingTrans().findMappingInput();
            for (MappingInput input : inputs) {
                for (RowSet rs : input.getInputRowSets()) {
                    // 
                    if (rs.getOriginStepName().equalsIgnoreCase(from)) {
                        return rs;
                    }
                }
            }
        }
    }
    // 
    return null;
}

Example 5 with MappingInput

use of org.pentaho.di.trans.steps.mappinginput.MappingInput in project pentaho-kettle by pentaho.

the class Trans method findMappingInput.

/**
 * Gets the mapping inputs for each step in the transformation.
 *
 * @return an array of MappingInputs
 */
public MappingInput[] findMappingInput() {
    if (steps == null) {
        return null;
    }
    List<MappingInput> list = new ArrayList<>();
    // Look in threads and find the MappingInput step thread...
    for (int i = 0; i < steps.size(); i++) {
        StepMetaDataCombi smdc = steps.get(i);
        StepInterface step = smdc.step;
        if (step.getStepID().equalsIgnoreCase("MappingInput")) {
            list.add((MappingInput) step);
        }
    }
    return list.toArray(new MappingInput[list.size()]);
}