Examples with StepInterface org.pentaho.di.trans.step.StepInterface used on opensource projects

Example 16 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class Mapping method pickupTargetStepsFor.

@VisibleForTesting
StepInterface[] pickupTargetStepsFor(MappingIODefinition outputDefinition) throws KettleException {
    List<StepInterface> result;
    if (!Utils.isEmpty(outputDefinition.getOutputStepname())) {
        // If we have a target step specification for the output of the mapping,
        // we need to send it over there...
        // 
        result = getTrans().findStepInterfaces(outputDefinition.getOutputStepname());
        if (Utils.isEmpty(result)) {
            throw new KettleException(BaseMessages.getString(PKG, "MappingDialog.Exception.StepNameNotFound", outputDefinition.getOutputStepname()));
        }
    } else {
        // No target step is specified.
        // See if we can find the next steps in the transformation..
        // 
        List<StepMeta> nextSteps = getTransMeta().findNextSteps(getStepMeta());
        // Let's send the data to all the next steps we find...
        // The origin is the mapping output step
        // The target is all the next steps after this mapping step.
        // 
        result = new ArrayList<>();
        for (StepMeta nextStep : nextSteps) {
            // need to take into the account different copies of the step
            List<StepInterface> copies = getTrans().findStepInterfaces(nextStep.getName());
            if (copies != null) {
                result.addAll(copies);
            }
        }
    }
    return result.toArray(new StepInterface[result.size()]);
}

Example 17 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class BlockUntilStepsFinish method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (BlockUntilStepsFinishMeta) smi;
    data = (BlockUntilStepsFinishData) sdi;
    if (first) {
        first = false;
        String[] stepnames = null;
        int stepnrs = 0;
        if (meta.getStepName() != null && meta.getStepName().length > 0) {
            stepnames = meta.getStepName();
            stepnrs = stepnames.length;
        } else {
            throw new KettleException(BaseMessages.getString(PKG, "BlockUntilStepsFinish.Error.NotSteps"));
        }
        // Get target stepnames
        String[] targetSteps = getTransMeta().getNextStepNames(getStepMeta());
        data.stepInterfaces = new ConcurrentHashMap<Integer, StepInterface>();
        for (int i = 0; i < stepnrs; i++) {
            // We can not get metrics from current step
            if (stepnames[i].equals(getStepname())) {
                throw new KettleException("You can not wait for step [" + stepnames[i] + "] to finish!");
            }
            if (targetSteps != null) {
                // We can not metrics from the target steps
                for (int j = 0; j < targetSteps.length; j++) {
                    if (stepnames[i].equals(targetSteps[j])) {
                        throw new KettleException("You can not get metrics for the target step [" + targetSteps[j] + "]!");
                    }
                }
            }
            int CopyNr = Const.toInt(meta.getStepCopyNr()[i], 0);
            StepInterface step = getDispatcher().findBaseSteps(stepnames[i]).get(CopyNr);
            if (step == null) {
                throw new KettleException("Erreur finding step [" + stepnames[i] + "] nr copy=" + CopyNr + "!");
            }
            data.stepInterfaces.put(i, getDispatcher().findBaseSteps(stepnames[i]).get(CopyNr));
        }
    }
    // Wait until all specified steps have finished!
    while (data.continueLoop && !isStopped()) {
        data.continueLoop = false;
        Iterator<Entry<Integer, StepInterface>> it = data.stepInterfaces.entrySet().iterator();
        while (it.hasNext()) {
            Entry<Integer, StepInterface> e = it.next();
            StepInterface step = e.getValue();
            if (step.getStatus() != StepExecutionStatus.STATUS_FINISHED) {
                // This step is still running...
                data.continueLoop = true;
            } else {
                // We have done with this step.
                // remove it from the map
                data.stepInterfaces.remove(e.getKey());
                if (log.isDetailed()) {
                    logDetailed("Finished running step [" + step.getStepname() + "(" + step.getCopy() + ")].");
                }
            }
        }
        if (data.continueLoop) {
            try {
                Thread.sleep(200);
            } catch (Exception e) {
            // ignore
            }
        }
    }
    // All steps we are waiting for are ended
    // let's now free all incoming rows
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    putRow(getInputRowMeta(), r);
    return true;
}

Example 18 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class JsonOutputTest method test.

public String test(boolean compatibilityMode) throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testJsonOutput");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testJsonOutput1", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the json output step
    // but first lets get a filename
    String jsonFileName = TestUtilities.createEmptyTempFile("testJsonOutput1_");
    StepMeta jsonOutputStep = createJsonOutputStep("json output step", jsonFileName, registry);
    ((JsonOutputMeta) jsonOutputStep.getStepMetaInterface()).setCompatibilityMode(compatibilityMode);
    transMeta.addStep(jsonOutputStep);
    // create a TransHopMeta for jsonOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, jsonOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Create a dummy step and add it to the tranMeta
    String dummyStepName = "dummy step";
    StepMeta dummyStep = createDummyStep(dummyStepName, registry);
    transMeta.addStep(dummyStep);
    // create a TransHopMeta for the
    TransHopMeta hop_outputJson_dummyStep = new TransHopMeta(jsonOutputStep, dummyStep);
    transMeta.addTransHop(hop_outputJson_dummyStep);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(dummyStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    trans.waitUntilFinished();
    // get the results and return it
    File outputFile = new File(jsonFileName + ".js");
    String jsonStructure = FileUtils.readFileToString(outputFile);
    return jsonStructure;
}

Example 19 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class MappingUnitTest method pickupTargetStepsFor_OutputIsNotDefined.

@SuppressWarnings("unchecked")
@Test
public void pickupTargetStepsFor_OutputIsNotDefined() throws Exception {
    StepMeta singleMeta = new StepMeta("single", null);
    StepMeta copiedMeta = new StepMeta("copied", null);
    Mockito.when(mockHelper.transMeta.findNextSteps(mockHelper.stepMeta)).thenReturn(Arrays.asList(singleMeta, copiedMeta));
    StepInterface single = Mockito.mock(StepInterface.class);
    Mockito.when(mockHelper.trans.findStepInterfaces("single")).thenReturn(Collections.singletonList(single));
    StepInterface copy1 = Mockito.mock(StepInterface.class);
    StepInterface copy2 = Mockito.mock(StepInterface.class);
    Mockito.when(mockHelper.trans.findStepInterfaces("copied")).thenReturn(Arrays.asList(copy1, copy2));
    MappingIODefinition definition = new MappingIODefinition(null, null);
    StepInterface[] targetSteps = mapping.pickupTargetStepsFor(definition);
    assertThat(Arrays.asList(targetSteps), JUnitMatchers.hasItems(is(single), is(copy1), is(copy2)));
}

Example 20 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class MappingInputTest method testSetConnectorSteps.

@Test
public void testSetConnectorSteps() throws Exception {
    when(stepMockHelper.transMeta.getSizeRowset()).thenReturn(1);
    MappingInputData mappingInputData = new MappingInputData();
    MappingInput mappingInput = new MappingInput(stepMockHelper.stepMeta, mappingInputData, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    mappingInput.init(stepMockHelper.initStepMetaInterface, mappingInputData);
    ValidatorData validatorData = new ValidatorData();
    Validator previousStep = new Validator(stepMockHelper.stepMeta, validatorData, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    when(stepMockHelper.stepMeta.isDoingErrorHandling()).thenReturn(true);
    StepErrorMeta stepErrorMeta = mock(StepErrorMeta.class);
    when(stepErrorMeta.getTargetStep()).thenReturn(stepMockHelper.stepMeta);
    when(stepMockHelper.stepMeta.getName()).thenReturn(stepName);
    when(stepMockHelper.stepMeta.getStepErrorMeta()).thenReturn(stepErrorMeta);
    StepInterface[] si = new StepInterface[] { previousStep };
    mappingInput.setConnectorSteps(si, Collections.<MappingValueRename>emptyList(), stepName);
    assertEquals(previousStep.getOutputRowSets().size(), 0);
}