Examples with SingleThreadedTransExecutor org.pentaho.di.trans.SingleThreadedTransExecutor used on opensource projects

Example 1 with SingleThreadedTransExecutor

use of org.pentaho.di.trans.SingleThreadedTransExecutor in project pentaho-kettle by pentaho.

the class SingleThreader method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    SingleThreaderData singleThreaderData = getData();
    // Set the type to single threaded in case the user forgot...
    // 
    singleThreaderData.mappingTransMeta.setTransformationType(TransformationType.SingleThreaded);
    // Create the transformation from meta-data...
    singleThreaderData.mappingTrans = new Trans(singleThreaderData.mappingTransMeta, getTrans());
    // Pass the parameters down to the sub-transformation.
    // 
    StepWithMappingMeta.activateParams(getData().mappingTrans, getData().mappingTrans, this, getData().mappingTrans.listParameters(), meta.getParameters(), meta.getParameterValues());
    getData().mappingTrans.activateParameters();
    // Disable thread priority managment as it will slow things down needlessly.
    // The single threaded engine doesn't use threads and doesn't need row locking.
    // 
    singleThreaderData.mappingTrans.getTransMeta().setUsingThreadPriorityManagment(false);
    // Leave a path up so that we can set variables in sub-transformations...
    // 
    singleThreaderData.mappingTrans.setParentTrans(getTrans());
    // Pass down the safe mode flag to the mapping...
    // 
    singleThreaderData.mappingTrans.setSafeModeEnabled(getTrans().isSafeModeEnabled());
    // Pass down the metrics gathering flag to the mapping...
    // 
    singleThreaderData.mappingTrans.setGatheringMetrics(getTrans().isGatheringMetrics());
    // Also set the name of this step in the mapping transformation for logging purposes
    // 
    singleThreaderData.mappingTrans.setMappingStepName(getStepname());
    initServletConfig();
    // prepare the execution
    // 
    singleThreaderData.mappingTrans.prepareExecution(null);
    // 
    if (singleThreaderData.injectStepMeta.isMappingInput()) {
        MappingInputData mappingInputData = (MappingInputData) singleThreaderData.mappingTrans.findDataInterface(singleThreaderData.injectStepMeta.getName());
        mappingInputData.sourceSteps = new StepInterface[0];
        mappingInputData.valueRenames = new ArrayList<MappingValueRename>();
    }
    // Add row producer & row listener
    singleThreaderData.rowProducer = singleThreaderData.mappingTrans.addRowProducer(meta.getInjectStep(), 0);
    StepInterface retrieveStep = singleThreaderData.mappingTrans.getStepInterface(meta.getRetrieveStep(), 0);
    retrieveStep.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
            // Simply pass it along to the next steps after the SingleThreader
            // 
            SingleThreader.this.putRow(rowMeta, row);
        }
    });
    singleThreaderData.mappingTrans.startThreads();
    // Create the executor...
    singleThreaderData.executor = new SingleThreadedTransExecutor(singleThreaderData.mappingTrans);
    // 
    try {
        boolean ok = singleThreaderData.executor.init();
        if (!ok) {
            throw new KettleException(BaseMessages.getString(PKG, "SingleThreader.Exception.UnableToInitSingleThreadedTransformation"));
        }
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "SingleThreader.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // Add the mapping transformation to the active sub-transformations map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), singleThreaderData.mappingTrans);
}

Example 2 with SingleThreadedTransExecutor

use of org.pentaho.di.trans.SingleThreadedTransExecutor in project pentaho-kettle by pentaho.

the class Mapping method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    initTransFromMeta();
    MappingData mappingData = getData();
    // 
    try {
        mappingData.getMappingTrans().prepareExecution(getTrans().getArguments());
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // 
    switch(mappingData.mappingTransMeta.getTransformationType()) {
        case Normal:
        case SerialSingleThreaded:
            break;
        case SingleThreaded:
            mappingData.singleThreadedTransExcecutor = new SingleThreadedTransExecutor(mappingData.getMappingTrans());
            if (!mappingData.singleThreadedTransExcecutor.init()) {
                throw new KettleException(BaseMessages.getString(PKG, "Mapping.Exception.UnableToInitSingleThreadedTransformation"));
            }
            break;
        default:
            break;
    }
    // If there is no read/write logging step set, we can insert the data from
    // the first mapping input/output step...
    // 
    MappingInput[] mappingInputs = mappingData.getMappingTrans().findMappingInput();
    LogTableField readField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_READ);
    if (readField.getSubject() == null && mappingInputs != null && mappingInputs.length >= 1) {
        readField.setSubject(mappingInputs[0].getStepMeta());
    }
    MappingOutput[] mappingOutputs = mappingData.getMappingTrans().findMappingOutput();
    LogTableField writeField = mappingData.mappingTransMeta.getTransLogTable().findField(TransLogTable.ID.LINES_WRITTEN);
    if (writeField.getSubject() == null && mappingOutputs != null && mappingOutputs.length >= 1) {
        writeField.setSubject(mappingOutputs[0].getStepMeta());
    }
    // Before we add rowsets and all, we should note that the mapping step did
    // not receive ANY input and output rowsets.
    // This is an exception to the general rule, built into
    // Trans.prepareExecution()
    // 
    // A Mapping Input step is supposed to read directly from the previous
    // steps.
    // A Mapping Output step is supposed to write directly to the next steps.
    // OK, check the input mapping definitions and look up the steps to read
    // from.
    // 
    StepInterface[] sourceSteps;
    for (MappingIODefinition inputDefinition : meta.getInputMappings()) {
        // 
        if (!Utils.isEmpty(inputDefinition.getInputStepname())) {
            StepInterface sourceStep = getTrans().findRunThread(inputDefinition.getInputStepname());
            if (sourceStep == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getInputStepname()));
            }
            sourceSteps = new StepInterface[] { sourceStep };
        } else {
            // We have no defined source step.
            // That means that we're reading from all input steps that this mapping
            // step has.
            // 
            List<StepMeta> prevSteps = getTransMeta().findPreviousSteps(getStepMeta());
            // TODO: Handle remote steps from: getStepMeta().getRemoteInputSteps()
            // 
            // Let's read data from all the previous steps we find...
            // The origin is the previous step
            // The target is the Mapping Input step.
            // 
            sourceSteps = new StepInterface[prevSteps.size()];
            for (int s = 0; s < sourceSteps.length; s++) {
                sourceSteps[s] = getTrans().findRunThread(prevSteps.get(s).getName());
            }
        }
        // What step are we writing to?
        MappingInput mappingInputTarget = null;
        MappingInput[] mappingInputSteps = mappingData.getMappingTrans().findMappingInput();
        if (Utils.isEmpty(inputDefinition.getOutputStepname())) {
            if (mappingInputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingInputStepRequired"));
            }
            if (mappingInputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingInputStepAllowed", "" + mappingInputSteps.length));
            }
            mappingInputTarget = mappingInputSteps[0];
        } else {
            // A target step was specified. See if we can find it...
            for (int s = 0; s < mappingInputSteps.length && mappingInputTarget == null; s++) {
                if (mappingInputSteps[s].getStepname().equals(inputDefinition.getOutputStepname())) {
                    mappingInputTarget = mappingInputSteps[s];
                }
            }
            // If we still didn't find it it's a drag.
            if (mappingInputTarget == null) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", inputDefinition.getOutputStepname()));
            }
        }
        // 
        if (inputDefinition.isRenamingOnOutput()) {
            addInputRenames(getData().inputRenameList, inputDefinition.getValueRenames());
        }
        mappingInputTarget.setConnectorSteps(sourceSteps, inputDefinition.getValueRenames(), getStepname());
    }
    // 
    for (MappingIODefinition outputDefinition : meta.getOutputMappings()) {
        // OK, what is the source (input) step in the mapping: it's the mapping
        // output step...
        // What step are we reading from here?
        // 
        MappingOutput mappingOutputSource = (MappingOutput) mappingData.getMappingTrans().findRunThread(outputDefinition.getInputStepname());
        if (mappingOutputSource == null) {
            // No source step was specified: we're reading from a single Mapping
            // Output step.
            // We should verify this if this is really the case...
            // 
            MappingOutput[] mappingOutputSteps = mappingData.getMappingTrans().findMappingOutput();
            if (mappingOutputSteps.length == 0) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OneMappingOutputStepRequired"));
            }
            if (mappingOutputSteps.length > 1) {
                throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.OnlyOneMappingOutputStepAllowed", "" + mappingOutputSteps.length));
            }
            mappingOutputSource = mappingOutputSteps[0];
        }
        // To what steps in this transformation are we writing to?
        // 
        StepInterface[] targetSteps = pickupTargetStepsFor(outputDefinition);
        // Now tell the mapping output step where to look...
        // Also explain the mapping output steps how to rename the values back...
        // 
        mappingOutputSource.setConnectorSteps(targetSteps, getData().inputRenameList, outputDefinition.getValueRenames());
        // Is this mapping copying or distributing?
        // Make sure the mapping output step mimics this behavior:
        // 
        mappingOutputSource.setDistributed(isDistributed());
    }
    // Finally, add the mapping transformation to the active sub-transformations
    // map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), getData().getMappingTrans());
}

Example 3 with SingleThreadedTransExecutor

use of org.pentaho.di.trans.SingleThreadedTransExecutor in project pentaho-kettle by pentaho.

the class SingleThreadedExecutionGuarder method failsWhenGivenNonSingleThreadSteps.

@Test(expected = KettleException.class)
public void failsWhenGivenNonSingleThreadSteps() throws Exception {
    Meta metaInterface = createMeta();
    PluginRegistry plugReg = PluginRegistry.getInstance();
    String id = plugReg.getPluginId(StepPluginType.class, metaInterface);
    assertNotNull("pluginId", id);
    StepMeta stepMeta = new StepMeta(id, "stepMetrics", metaInterface);
    stepMeta.setDraw(true);
    TransMeta transMeta = new TransMeta();
    transMeta.setName("failsWhenGivenNonSingleThreadSteps");
    transMeta.addStep(stepMeta);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor(trans);
    executor.init();
}