Examples with StepInterface org.pentaho.di.trans.step.StepInterface used on opensource projects

Example 86 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class SimpleMapping method processRow.

/**
 * Process a single row. In our case, we send one row of data to a piece of transformation. In the transformation, we
 * look up the MappingInput step to send our rows to it. As a consequence, for the time being, there can only be one
 * MappingInput and one MappingOutput step in the Mapping.
 */
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (SimpleMappingMeta) smi;
    setData((SimpleMappingData) sdi);
    SimpleMappingData simpleMappingData = getData();
    try {
        if (first) {
            first = false;
            simpleMappingData.wasStarted = true;
            // Rows read are injected into the one available Mapping Input step
            // 
            String mappingInputStepname = simpleMappingData.mappingInput.getStepname();
            RowProducer rowProducer = simpleMappingData.mappingTrans.addRowProducer(mappingInputStepname, 0);
            simpleMappingData.rowDataInputMapper = new RowDataInputMapper(meta.getInputMapping(), rowProducer);
            // Rows produced by the mapping are read and passed on.
            // 
            String mappingOutputStepname = simpleMappingData.mappingOutput.getStepname();
            StepInterface outputStepInterface = simpleMappingData.mappingTrans.findStepInterface(mappingOutputStepname, 0);
            RowOutputDataMapper outputDataMapper = new RowOutputDataMapper(meta.getInputMapping(), meta.getOutputMapping(), new PutRowInterface() {

                @Override
                public void putRow(RowMetaInterface rowMeta, Object[] rowData) throws KettleStepException {
                    SimpleMapping.this.putRow(rowMeta, rowData);
                }
            });
            outputStepInterface.addRowListener(outputDataMapper);
            // Start the mapping/sub-transformation threads
            // 
            simpleMappingData.mappingTrans.startThreads();
        }
        // The data we read we pass to the mapping
        // 
        Object[] row = getRow();
        boolean rowWasPut = false;
        if (row != null) {
            while (!(data.mappingTrans.isFinishedOrStopped() || rowWasPut)) {
                rowWasPut = data.rowDataInputMapper.putRow(getInputRowMeta(), row);
            }
        }
        if (!rowWasPut) {
            simpleMappingData.rowDataInputMapper.finished();
            simpleMappingData.mappingTrans.waitUntilFinished();
            setOutputDone();
            return false;
        }
        return true;
    } catch (Throwable t) {
        // 
        if (simpleMappingData.mappingTrans != null) {
            simpleMappingData.mappingTrans.stopAll();
        }
        // 
        throw new KettleException(t);
    }
}

Example 87 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class ScriptValuesAddedFunctions method setVariable.

public static void setVariable(Context actualContext, Scriptable actualObject, Object[] arguments, Function functionContext) {
    if (arguments.length != 3) {
        throw Context.reportRuntimeError("The function call setVariable requires 3 arguments.");
    }
    Object stepObject = Context.jsToJava(actualObject.get("_step_", actualObject), StepInterface.class);
    if (stepObject instanceof StepInterface) {
        StepInterface step = (StepInterface) stepObject;
        Trans trans = step.getTrans();
        final String variableName = Context.toString(arguments[0]);
        final String variableValue = Context.toString(arguments[1]);
        final VariableScope variableScope = getVariableScope(Context.toString(arguments[2]));
        // Set variable in step's scope so that it can be retrieved in the same step using getVariable
        step.setVariable(variableName, variableValue);
        switch(variableScope) {
            case PARENT:
                setParentScopeVariable(trans, variableName, variableValue);
                break;
            case GRAND_PARENT:
                setGrandParentScopeVariable(trans, variableName, variableValue);
                break;
            case ROOT:
                setRootScopeVariable(trans, variableName, variableValue);
                break;
            case SYSTEM:
                setSystemScopeVariable(trans, variableName, variableValue);
                break;
        }
    }
}

Example 88 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class ScriptValuesAddedFunctions method getProcessCount.

public static double getProcessCount(Context actualContext, Scriptable actualObject, Object[] ArgList, Function FunctionContext) {
    if (ArgList.length == 1) {
        try {
            Object scmO = actualObject.get("_step_", actualObject);
            StepInterface scm = (StepInterface) Context.jsToJava(scmO, StepInterface.class);
            String strType = Context.toString(ArgList[0]).toLowerCase();
            if (strType.equals("i")) {
                return scm.getLinesInput();
            } else if (strType.equals("o")) {
                return scm.getLinesOutput();
            } else if (strType.equals("r")) {
                return scm.getLinesRead();
            } else if (strType.equals("u")) {
                return scm.getLinesUpdated();
            } else if (strType.equals("w")) {
                return scm.getLinesWritten();
            } else if (strType.equals("e")) {
                return scm.getLinesRejected();
            } else {
                return 0;
            }
        } catch (Exception e) {
            // throw Context.reportRuntimeError(e.toString());
            return 0;
        }
    } else {
        throw Context.reportRuntimeError("The function call getProcessCount requires 1 argument.");
    }
}

Example 89 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class Mapping method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    initTransFromMeta();
    MappingData mappingData = getData();
    // 
    try {
        mappingData.getMappingTrans().prepareExecution(getTrans().getArguments());
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // 
    switch(mappingData.mappingTransMeta.getTransformationType()) {
        case Normal:
        case SerialSingleThreaded:
            break;
        case SingleThreaded:
            mappingData.singleThreadedTransExcecutor = new SingleThreadedTransExecutor(mappingData.getMappingTrans());
            if (!mappingData.singleThreadedTransExcecutor.init()) {
                throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToInitSingleThreadedTransformation"));
            }
            break;
        default:
            break;
    }
    // If there is no read/write logging step set, we can insert the data from
    // the first mapping input/output step...
    // 
    MappingInput[] mappingInputs = mappingData.getMappingTrans().findMappingInput();
    LogTableField readField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
    if (readField.getSubject() == null && mappingInputs != null && mappingInputs.length >= 1) {
        readField.setSubject(mappingInputs[0].getStepMeta());
    }
    MappingOutput[] mappingOutputs = mappingData.getMappingTrans().findMappingOutput();
    LogTableField writeField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
    if (writeField.getSubject() == null && mappingOutputs != null && mappingOutputs.length >= 1) {
        writeField.setSubject(mappingOutputs[0].getStepMeta());
    }
    // Before we add rowsets and all, we should note that the mapping step did
    // not receive ANY input and output rowsets.
    // This is an exception to the general rule, built into
    // Trans.prepareExecution()
    // 
    // A Mapping Input step is supposed to read directly from the previous
    // steps.
    // A Mapping Output step is supposed to write directly to the next steps.
    // OK, check the input mapping definitions and look up the steps to read
    // from.
    // 
    StepInterface[] sourceSteps;
    for (MappingIODefinition inputDefinition : meta.getInputMappings()) {
        // 
        if (!Utils.isEmpty(inputDefinition.getInputStepname())) {
            StepInterface sourceStep = getTrans().findRunThread(inputDefinition.getInputStepname());
            if (sourceStep == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getInputStepname()));
            }
            sourceSteps = new StepInterface[] { sourceStep };
        } else {
            // We have no defined source step.
            // That means that we're reading from all input steps that this mapping
            // step has.
            // 
            List<StepMeta> prevSteps = getTransMeta().findPreviousSteps(getStepMeta());
            // TODO: Handle remote steps from: getStepMeta().getRemoteInputSteps()
            // 
            // Let's read data from all the previous steps we find...
            // The origin is the previous step
            // The target is the Mapping Input step.
            // 
            sourceSteps = new StepInterface[prevSteps.size()];
            for (int s = 0; s < sourceSteps.length; s++) {
                sourceSteps[s] = getTrans().findRunThread(prevSteps.get(s).getName());
            }
        }
        // What step are we writing to?
        MappingInput mappingInputTarget = null;
        MappingInput[] mappingInputSteps = mappingData.getMappingTrans().findMappingInput();
        if (Utils.isEmpty(inputDefinition.getOutputStepname())) {
            if (mappingInputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingInputStepRequired"));
            }
            if (mappingInputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingInputStepAllowed", "" + mappingInputSteps.length));
            }
            mappingInputTarget = mappingInputSteps[0];
        } else {
            // A target step was specified. See if we can find it...
            for (int s = 0; s < mappingInputSteps.length && mappingInputTarget == null; s++) {
                if (mappingInputSteps[s].getStepname().equals(inputDefinition.getOutputStepname())) {
                    mappingInputTarget = mappingInputSteps[s];
                }
            }
            // If we still didn't find it it's a drag.
            if (mappingInputTarget == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getOutputStepname()));
            }
        }
        // 
        if (inputDefinition.isRenamingOnOutput()) {
            addInputRenames(getData().inputRenameList, inputDefinition.getValueRenames());
        }
        mappingInputTarget.setConnectorSteps(sourceSteps, inputDefinition.getValueRenames(), getStepname());
    }
    // 
    for (MappingIODefinition outputDefinition : meta.getOutputMappings()) {
        // OK, what is the source (input) step in the mapping: it's the mapping
        // output step...
        // What step are we reading from here?
        // 
        MappingOutput mappingOutputSource = (MappingOutput) mappingData.getMappingTrans().findRunThread(outputDefinition.getInputStepname());
        if (mappingOutputSource == null) {
            // No source step was specified: we're reading from a single Mapping
            // Output step.
            // We should verify this if this is really the case...
            // 
            MappingOutput[] mappingOutputSteps = mappingData.getMappingTrans().findMappingOutput();
            if (mappingOutputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingOutputStepRequired"));
            }
            if (mappingOutputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingOutputStepAllowed", "" + mappingOutputSteps.length));
            }
            mappingOutputSource = mappingOutputSteps[0];
        }
        // To what steps in this transformation are we writing to?
        // 
        StepInterface[] targetSteps = pickupTargetStepsFor(outputDefinition);
        // Now tell the mapping output step where to look...
        // Also explain the mapping output steps how to rename the values back...
        // 
        mappingOutputSource.setConnectorSteps(targetSteps, getData().inputRenameList, outputDefinition.getValueRenames());
        // Is this mapping copying or distributing?
        // Make sure the mapping output step mimics this behavior:
        // 
        mappingOutputSource.setDistributed(isDistributed());
    }
    // Finally, add the mapping transformation to the active sub-transformations
    // map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), getData().getMappingTrans());
}

Example 90 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class TransPainter method checkDrawSlowStepIndicator.

private void checkDrawSlowStepIndicator(StepMeta stepMeta) {
    if (stepMeta == null) {
        return;
    }
    // draw optional performance indicator
    if (trans != null) {
        Point pt = stepMeta.getLocation();
        if (pt == null) {
            pt = new Point(50, 50);
        }
        Point screen = real2screen(pt.x, pt.y);
        int x = screen.x;
        int y = screen.y;
        List<StepInterface> steps = trans.findBaseSteps(stepMeta.getName());
        for (StepInterface step : steps) {
            if (step.isRunning()) {
                int inputRows = step.rowsetInputSize();
                int outputRows = step.rowsetOutputSize();
                // if the step can't keep up with its input, mark it by drawing an animation
                boolean isSlow = inputRows * 0.85 > outputRows;
                if (isSlow) {
                    gc.setLineWidth(linewidth + 1);
                    if (System.currentTimeMillis() % 2000 > 1000) {
                        gc.setForeground(EColor.BACKGROUND);
                        gc.setLineStyle(ELineStyle.SOLID);
                        gc.drawRectangle(x + 1, y + 1, iconsize - 2, iconsize - 2);
                        gc.setForeground(EColor.DARKGRAY);
                        gc.setLineStyle(ELineStyle.DOT);
                        gc.drawRectangle(x + 1, y + 1, iconsize - 2, iconsize - 2);
                    } else {
                        gc.setForeground(EColor.DARKGRAY);
                        gc.setLineStyle(ELineStyle.SOLID);
                        gc.drawRectangle(x + 1, y + 1, iconsize - 2, iconsize - 2);
                        gc.setForeground(EColor.BACKGROUND);
                        gc.setLineStyle(ELineStyle.DOT);
                        gc.drawRectangle(x + 1, y + 1, iconsize - 2, iconsize - 2);
                    }
                }
            }
            gc.setLineStyle(ELineStyle.SOLID);
        }
    }
}