Examples with StepMetaInterface org.pentaho.di.trans.step.StepMetaInterface used on opensource projects

Example 66 with StepMetaInterface

use of org.pentaho.di.trans.step.StepMetaInterface in project pentaho-kettle by pentaho.

the class MetaInject method init.

public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (MetaInjectMeta) smi;
    data = (MetaInjectData) sdi;
    if (super.init(smi, sdi)) {
        try {
            meta.actualizeMetaInjectMapping();
            data.transMeta = loadTransformationMeta();
            data.transMeta.copyVariablesFrom(this);
            data.transMeta.mergeParametersWith(this.getTrans(), true);
            checkSoureStepsAvailability();
            checkTargetStepsAvailability();
            // Get a mapping between the step name and the injection...
            // 
            // Get new injection info
            data.stepInjectionMetasMap = new HashMap<String, StepMetaInterface>();
            for (StepMeta stepMeta : data.transMeta.getUsedSteps()) {
                StepMetaInterface meta = stepMeta.getStepMetaInterface();
                if (BeanInjectionInfo.isInjectionSupported(meta.getClass())) {
                    data.stepInjectionMetasMap.put(stepMeta.getName(), meta);
                }
            }
            // Get old injection info
            data.stepInjectionMap = new HashMap<String, StepMetaInjectionInterface>();
            for (StepMeta stepMeta : data.transMeta.getUsedSteps()) {
                StepMetaInjectionInterface injectionInterface = stepMeta.getStepMetaInterface().getStepMetaInjectionInterface();
                if (injectionInterface != null) {
                    data.stepInjectionMap.put(stepMeta.getName(), injectionInterface);
                }
            }
            // 
            if (meta.getStreamSourceStep() != null && !Utils.isEmpty(meta.getStreamTargetStepname())) {
                data.streaming = true;
                data.streamingSourceStepname = meta.getStreamSourceStep().getName();
                data.streamingTargetStepname = meta.getStreamTargetStepname();
            }
            return true;
        } catch (Exception e) {
            logError(BaseMessages.getString(PKG, "MetaInject.BadEncoding.Message"), e);
            return false;
        }
    }
    return false;
}

Example 67 with StepMetaInterface

use of org.pentaho.di.trans.step.StepMetaInterface in project pentaho-kettle by pentaho.

the class MetaInject method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MetaInjectMeta) smi;
    data = (MetaInjectData) sdi;
    // Read the data from all input steps and keep it in memory...
    // Skip the step from which we stream data. Keep that available for runtime action.
    // 
    data.rowMap = new HashMap<String, List<RowMetaAndData>>();
    for (String prevStepName : getTransMeta().getPrevStepNames(getStepMeta())) {
        // 
        if (!data.streaming || !prevStepName.equalsIgnoreCase(data.streamingSourceStepname)) {
            List<RowMetaAndData> list = new ArrayList<RowMetaAndData>();
            RowSet rowSet = findInputRowSet(prevStepName);
            Object[] row = getRowFrom(rowSet);
            while (row != null) {
                RowMetaAndData rd = new RowMetaAndData();
                rd.setRowMeta(rowSet.getRowMeta());
                rd.setData(row);
                list.add(rd);
                row = getRowFrom(rowSet);
            }
            if (!list.isEmpty()) {
                data.rowMap.put(prevStepName, list);
            }
        }
    }
    List<StepMeta> steps = data.transMeta.getSteps();
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjection(en.getKey(), en.getValue());
    }
    /*
     * constants injection should be executed after steps, because if constant should be inserted into target with array
     * in path, constants should be inserted into all arrays items
     */
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjectionConstants(en.getKey(), en.getValue());
    }
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        en.getValue().searchInfoAndTargetSteps(steps);
    }
    for (String targetStepName : data.stepInjectionMap.keySet()) {
        if (!data.stepInjectionMetasMap.containsKey(targetStepName)) {
            oldInjection(targetStepName);
            StepMeta targetStep = StepMeta.findStep(steps, targetStepName);
            if (targetStep != null) {
                targetStep.getStepMetaInterface().searchInfoAndTargetSteps(steps);
            }
        }
    }
    if (!meta.isNoExecution()) {
        // Now we can execute this modified transformation metadata.
        // 
        final Trans injectTrans = createInjectTrans();
        injectTrans.setMetaStore(getMetaStore());
        if (getTrans().getParentJob() != null) {
            // See PDI-13224
            injectTrans.setParentJob(getTrans().getParentJob());
        }
        getTrans().addTransStoppedListener(new TransStoppedListener() {

            public void transStopped(Trans parentTrans) {
                injectTrans.stopAll();
            }
        });
        injectTrans.prepareExecution(null);
        // See if we need to stream some data over...
        // 
        RowProducer rowProducer = null;
        if (data.streaming) {
            rowProducer = injectTrans.addRowProducer(data.streamingTargetStepname, 0);
        }
        // Finally, add the mapping transformation to the active sub-transformations
        // map in the parent transformation
        // 
        getTrans().addActiveSubTransformation(getStepname(), injectTrans);
        if (!Utils.isEmpty(meta.getSourceStepName())) {
            StepInterface stepInterface = injectTrans.getStepInterface(meta.getSourceStepName(), 0);
            if (stepInterface == null) {
                throw new KettleException("Unable to find step '" + meta.getSourceStepName() + "' to read from.");
            }
            stepInterface.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    // Just pass along the data as output of this step...
                    // 
                    MetaInject.this.putRow(rowMeta, row);
                }
            });
        }
        injectTrans.startThreads();
        if (data.streaming) {
            // Deplete all the rows from the parent transformation into the modified transformation
            // 
            RowSet rowSet = findInputRowSet(data.streamingSourceStepname);
            if (rowSet == null) {
                throw new KettleException("Unable to find step '" + data.streamingSourceStepname + "' to stream data from");
            }
            Object[] row = getRowFrom(rowSet);
            while (row != null && !isStopped()) {
                rowProducer.putRow(rowSet.getRowMeta(), row);
                row = getRowFrom(rowSet);
            }
            rowProducer.finished();
        }
        // 
        while (!injectTrans.isFinished() && !injectTrans.isStopped() && !isStopped()) {
            copyResult(injectTrans);
            // Wait a little bit.
            try {
                Thread.sleep(50);
            } catch (Exception e) {
            // Ignore errors
            }
        }
        copyResult(injectTrans);
        waitUntilFinished(injectTrans);
    }
    // let the transformation complete it's execution to allow for any customizations to MDI to happen in the init methods of steps
    if (log.isDetailed()) {
        logDetailed("XML of transformation after injection: " + data.transMeta.getXML());
    }
    String targetFile = environmentSubstitute(meta.getTargetFile());
    if (!Utils.isEmpty(targetFile)) {
        writeInjectedKtr(targetFile);
    }
    // All done!
    setOutputDone();
    return false;
}

Example 68 with StepMetaInterface

use of org.pentaho.di.trans.step.StepMetaInterface in project pentaho-kettle by pentaho.

the class GPBulkLoaderDialog method generateMappings.

/**
 * Reads in the fields from the previous steps and from the ONE next step and opens an EnterMappingDialog with this
 * information. After the user did the mapping, those information is put into the Select/Rename table.
 */
private void generateMappings() {
    // Determine the source and target fields...
    // 
    RowMetaInterface sourceFields;
    RowMetaInterface targetFields;
    try {
        sourceFields = transMeta.getPrevStepFields(stepMeta);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Title"), BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.UnableToFindSourceFields.Message"), e);
        return;
    }
    // refresh data
    input.setDatabaseMeta(transMeta.findDatabase(wConnection.getText()));
    input.setTableName(transMeta.environmentSubstitute(wTable.getText()));
    StepMetaInterface stepMetaInterface = stepMeta.getStepMetaInterface();
    try {
        targetFields = stepMetaInterface.getRequiredFields(transMeta);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Title"), BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Message"), e);
        return;
    }
    String[] inputNames = new String[sourceFields.size()];
    for (int i = 0; i < sourceFields.size(); i++) {
        ValueMetaInterface value = sourceFields.getValueMeta(i);
        inputNames[i] = value.getName() + EnterMappingDialog.STRING_ORIGIN_SEPARATOR + value.getOrigin() + ")";
    }
    // Create the existing mapping list...
    // 
    List<SourceToTargetMapping> mappings = new ArrayList<SourceToTargetMapping>();
    StringBuilder missingSourceFields = new StringBuilder();
    StringBuilder missingTargetFields = new StringBuilder();
    int nrFields = wReturn.nrNonEmpty();
    for (int i = 0; i < nrFields; i++) {
        TableItem item = wReturn.getNonEmpty(i);
        String source = item.getText(2);
        String target = item.getText(1);
        int sourceIndex = sourceFields.indexOfValue(source);
        if (sourceIndex < 0) {
            missingSourceFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        int targetIndex = targetFields.indexOfValue(target);
        if (targetIndex < 0) {
            missingTargetFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        if (sourceIndex < 0 || targetIndex < 0) {
            continue;
        }
        SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex);
        mappings.add(mapping);
    }
    // 
    if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) {
        String message = "";
        if (missingSourceFields.length() > 0) {
            message += BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.SomeSourceFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        if (missingTargetFields.length() > 0) {
            message += BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.SomeTargetFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        message += Const.CR;
        message += BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.SomeFieldsNotFoundContinue") + Const.CR;
        MessageDialog.setDefaultImage(GUIResource.getInstance().getImageSpoon());
        boolean goOn = MessageDialog.openConfirm(shell, BaseMessages.getString(PKG, "GPBulkLoaderDialog.DoMapping.SomeFieldsNotFoundTitle"), message);
        if (!goOn) {
            return;
        }
    }
    EnterMappingDialog d = new EnterMappingDialog(GPBulkLoaderDialog.this.shell, sourceFields.getFieldNames(), targetFields.getFieldNames(), mappings);
    mappings = d.open();
    // 
    if (mappings != null) {
        // Clear and re-populate!
        // 
        wReturn.table.removeAll();
        wReturn.table.setItemCount(mappings.size());
        for (int i = 0; i < mappings.size(); i++) {
            SourceToTargetMapping mapping = mappings.get(i);
            TableItem item = wReturn.table.getItem(i);
            item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
            item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName());
        }
        wReturn.setRowNums();
        wReturn.optWidth(true);
    }
}

Example 69 with StepMetaInterface

use of org.pentaho.di.trans.step.StepMetaInterface in project pentaho-kettle by pentaho.

the class GPLoadDialog method generateMappings.

/**
 * Reads in the fields from the previous steps and from the ONE next step and opens an EnterMappingDialog with this
 * information. After the user did the mapping, those information is put into the Select/Rename table.
 */
private void generateMappings() {
    // Determine the source and target fields...
    // 
    RowMetaInterface sourceFields;
    RowMetaInterface targetFields;
    try {
        sourceFields = transMeta.getPrevStepFields(stepMeta);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.UnableToFindSourceFields.Title"), BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.UnableToFindSourceFields.Message"), e);
        return;
    }
    // refresh data
    input.setDatabaseMeta(transMeta.findDatabase(wConnection.getText()));
    input.setTableName(transMeta.environmentSubstitute(wTable.getText()));
    StepMetaInterface stepMetaInterface = stepMeta.getStepMetaInterface();
    try {
        targetFields = stepMetaInterface.getRequiredFields(transMeta);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.UnableToFindTargetFields.Title"), BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.UnableToFindTargetFields.Message"), e);
        return;
    }
    String[] inputNames = new String[sourceFields.size()];
    for (int i = 0; i < sourceFields.size(); i++) {
        ValueMetaInterface value = sourceFields.getValueMeta(i);
        inputNames[i] = value.getName() + EnterMappingDialog.STRING_ORIGIN_SEPARATOR + value.getOrigin() + ")";
    }
    // Create the existing mapping list...
    // 
    List<SourceToTargetMapping> mappings = new ArrayList<SourceToTargetMapping>();
    StringBuilder missingSourceFields = new StringBuilder();
    StringBuilder missingTargetFields = new StringBuilder();
    int nrFields = wReturn.nrNonEmpty();
    for (int i = 0; i < nrFields; i++) {
        TableItem item = wReturn.getNonEmpty(i);
        String source = item.getText(2);
        String target = item.getText(1);
        int sourceIndex = sourceFields.indexOfValue(source);
        if (sourceIndex < 0) {
            missingSourceFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        int targetIndex = targetFields.indexOfValue(target);
        if (targetIndex < 0) {
            missingTargetFields.append(Const.CR).append("   ").append(source).append(" --> ").append(target);
        }
        if (sourceIndex < 0 || targetIndex < 0) {
            continue;
        }
        SourceToTargetMapping mapping = new SourceToTargetMapping(sourceIndex, targetIndex);
        mappings.add(mapping);
    }
    // 
    if (missingSourceFields.length() > 0 || missingTargetFields.length() > 0) {
        String message = "";
        if (missingSourceFields.length() > 0) {
            message += BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.SomeSourceFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        if (missingTargetFields.length() > 0) {
            message += BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.SomeTargetFieldsNotFound", missingSourceFields.toString()) + Const.CR;
        }
        message += Const.CR;
        message += BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.SomeFieldsNotFoundContinue") + Const.CR;
        MessageDialog.setDefaultImage(GUIResource.getInstance().getImageSpoon());
        boolean goOn = MessageDialog.openConfirm(shell, BaseMessages.getString(PKG, "GPLoadDialog.DoMapping.SomeFieldsNotFoundTitle"), message);
        if (!goOn) {
            return;
        }
    }
    EnterMappingDialog d = new EnterMappingDialog(GPLoadDialog.this.shell, sourceFields.getFieldNames(), targetFields.getFieldNames(), mappings);
    mappings = d.open();
    // 
    if (mappings != null) {
        // Clear and re-populate!
        // 
        wReturn.table.removeAll();
        wReturn.table.setItemCount(mappings.size());
        for (int i = 0; i < mappings.size(); i++) {
            SourceToTargetMapping mapping = mappings.get(i);
            TableItem item = wReturn.table.getItem(i);
            item.setText(2, sourceFields.getValueMeta(mapping.getSourcePosition()).getName());
            item.setText(1, targetFields.getValueMeta(mapping.getTargetPosition()).getName());
        }
        wReturn.setRowNums();
        wReturn.optWidth(true);
    }
}

Example 70 with StepMetaInterface

use of org.pentaho.di.trans.step.StepMetaInterface in project pentaho-kettle by pentaho.

the class MetaInjectTest method testGetUnavailableTargetKeys.

@Test
public void testGetUnavailableTargetKeys() throws Exception {
    final String targetStepName = "injectable step name";
    TargetStepAttribute unavailableTargetAttr = new TargetStepAttribute(targetStepName, "NOT_THERE", false);
    TargetStepAttribute availableTargetAttr = new TargetStepAttribute(targetStepName, "THERE", false);
    SourceStepField sourceStep = new SourceStepField(TEST_SOURCE_STEP_NAME, TEST_FIELD);
    Map<TargetStepAttribute, SourceStepField> targetMap = new HashMap<>(2);
    targetMap.put(unavailableTargetAttr, sourceStep);
    targetMap.put(availableTargetAttr, sourceStep);
    StepMetaInterface smi = new InjectableTestStepMeta();
    TransMeta transMeta = mockSingleStepTransMeta(targetStepName, smi);
    Set<TargetStepAttribute> unavailable = MetaInject.getUnavailableTargetKeys(targetMap, transMeta, Collections.<TargetStepAttribute>emptySet());
    assertEquals(1, unavailable.size());
    assertTrue(unavailable.contains(unavailableTargetAttr));
}