Examples with StreamInterface org.pentaho.di.trans.step.errorhandling.StreamInterface used on opensource projects

Example 56 with StreamInterface

use of org.pentaho.di.trans.step.errorhandling.StreamInterface in project pentaho-kettle by pentaho.

the class MergeRows method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MergeRowsMeta) smi;
    data = (MergeRowsData) sdi;
    if (first) {
        first = false;
        // Find the appropriate RowSet
        // 
        List<StreamInterface> infoStreams = meta.getStepIOMeta().getInfoStreams();
        // oneRowSet is the "Reference" stream
        data.oneRowSet = findInputRowSet(infoStreams.get(0).getStepname());
        // twoRowSet is the "Comparison" stream
        data.twoRowSet = findInputRowSet(infoStreams.get(1).getStepname());
        // rowSetWhenIdentical is use in case the comparison is IDENTICAL.
        // this should be the "Comparison" stream but can be the "Reference" stream for backward compatibility (PDI-736)
        String useRefWhenIdenticalVar = Const.NVL(System.getProperty(Const.KETTLE_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL), "N");
        if ("N".equalsIgnoreCase(useRefWhenIdenticalVar)) {
            // use the reference stream (as per documentation)
            useRefWhenIdentical = false;
        } else {
            // use the comparison stream (for backward compatibility)
            useRefWhenIdentical = true;
        }
        data.one = getRowFrom(data.oneRowSet);
        data.two = getRowFrom(data.twoRowSet);
        try {
            checkInputLayoutValid(data.oneRowSet.getRowMeta(), data.twoRowSet.getRowMeta());
        } catch (KettleRowException e) {
            throw new KettleException(BaseMessages.getString(PKG, "MergeRows.Exception.InvalidLayoutDetected"), e);
        }
        if (data.one != null) {
            // Find the key indexes:
            data.keyNrs = new int[meta.getKeyFields().length];
            for (int i = 0; i < data.keyNrs.length; i++) {
                data.keyNrs[i] = data.oneRowSet.getRowMeta().indexOfValue(meta.getKeyFields()[i]);
                if (data.keyNrs[i] < 0) {
                    String message = BaseMessages.getString(PKG, "MergeRows.Exception.UnableToFindFieldInReferenceStream", meta.getKeyFields()[i]);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
        }
        if (data.two != null) {
            data.valueNrs = new int[meta.getValueFields().length];
            for (int i = 0; i < data.valueNrs.length; i++) {
                data.valueNrs[i] = data.twoRowSet.getRowMeta().indexOfValue(meta.getValueFields()[i]);
                if (data.valueNrs[i] < 0) {
                    String message = BaseMessages.getString(PKG, "MergeRows.Exception.UnableToFindFieldInReferenceStream", meta.getValueFields()[i]);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
        }
    }
    if (log.isRowLevel()) {
        logRowlevel(BaseMessages.getString(PKG, "MergeRows.Log.DataInfo", data.one + "") + data.two);
    }
    if (data.one == null && data.two == null) {
        setOutputDone();
        return false;
    }
    if (data.outputRowMeta == null) {
        data.outputRowMeta = new RowMeta();
        if (data.one != null) {
            meta.getFields(data.outputRowMeta, getStepname(), new RowMetaInterface[] { data.oneRowSet.getRowMeta() }, null, this, repository, metaStore);
        } else {
            meta.getFields(data.outputRowMeta, getStepname(), new RowMetaInterface[] { data.twoRowSet.getRowMeta() }, null, this, repository, metaStore);
        }
    }
    Object[] outputRow;
    int outputIndex;
    String flagField = null;
    if (data.one == null && data.two != null) {
        // Record 2 is flagged as new!
        outputRow = data.two;
        outputIndex = data.twoRowSet.getRowMeta().size();
        flagField = VALUE_NEW;
        // Also get a next row from compare rowset...
        data.two = getRowFrom(data.twoRowSet);
    } else if (data.one != null && data.two == null) {
        // Record 1 is flagged as deleted!
        outputRow = data.one;
        outputIndex = data.oneRowSet.getRowMeta().size();
        flagField = VALUE_DELETED;
        // Also get a next row from reference rowset...
        data.one = getRowFrom(data.oneRowSet);
    } else {
        // OK, Here is the real start of the compare code!
        int compare = data.oneRowSet.getRowMeta().compare(data.one, data.two, data.keyNrs);
        if (compare == 0) {
            // The Key matches, we CAN compare the two rows...
            int compareValues = data.oneRowSet.getRowMeta().compare(data.one, data.two, data.valueNrs);
            if (compareValues == 0) {
                if (useRefWhenIdentical) {
                    // backwards compatible behavior: use the reference stream (PDI-736)
                    outputRow = data.one;
                    outputIndex = data.oneRowSet.getRowMeta().size();
                } else {
                    // documented behavior: use the comparison stream (PDI-736)
                    outputRow = data.two;
                    outputIndex = data.twoRowSet.getRowMeta().size();
                }
                flagField = VALUE_IDENTICAL;
            } else {
                // Return the compare (most recent) row
                // 
                outputRow = data.two;
                outputIndex = data.twoRowSet.getRowMeta().size();
                flagField = VALUE_CHANGED;
            }
            // Get a new row from both streams...
            data.one = getRowFrom(data.oneRowSet);
            data.two = getRowFrom(data.twoRowSet);
        } else {
            if (compare < 0) {
                // one < two
                outputRow = data.one;
                outputIndex = data.oneRowSet.getRowMeta().size();
                flagField = VALUE_DELETED;
                data.one = getRowFrom(data.oneRowSet);
            } else {
                outputRow = data.two;
                outputIndex = data.twoRowSet.getRowMeta().size();
                flagField = VALUE_NEW;
                data.two = getRowFrom(data.twoRowSet);
            }
        }
    }
    // send the row to the next steps...
    putRow(data.outputRowMeta, RowDataUtil.addValueData(outputRow, outputIndex, flagField));
    if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "MergeRows.LineNumber") + getLinesRead());
        }
    }
    return true;
}

Example 57 with StreamInterface

use of org.pentaho.di.trans.step.errorhandling.StreamInterface in project pentaho-kettle by pentaho.

the class MergeRowsMeta method saveRep.

@Override
public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException {
    try {
        for (int i = 0; i < keyFields.length; i++) {
            rep.saveStepAttribute(id_transformation, id_step, i, "key_field", keyFields[i]);
        }
        for (int i = 0; i < valueFields.length; i++) {
            rep.saveStepAttribute(id_transformation, id_step, i, "value_field", valueFields[i]);
        }
        rep.saveStepAttribute(id_transformation, id_step, "flag_field", flagField);
        List<StreamInterface> infoStreams = getStepIOMeta().getInfoStreams();
        StreamInterface referenceStream = infoStreams.get(0);
        StreamInterface compareStream = infoStreams.get(1);
        rep.saveStepAttribute(id_transformation, id_step, "reference", referenceStream.getStepname());
        rep.saveStepAttribute(id_transformation, id_step, "compare", compareStream.getStepname());
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "MergeRowsMeta.Exception.UnableToSaveStepInfo") + id_step, e);
    }
}

Example 58 with StreamInterface

use of org.pentaho.di.trans.step.errorhandling.StreamInterface in project pentaho-kettle by pentaho.

the class MergeRowsMeta method readData.

private void readData(Node stepnode) throws KettleXMLException {
    try {
        Node keysnode = XMLHandler.getSubNode(stepnode, "keys");
        Node valuesnode = XMLHandler.getSubNode(stepnode, "values");
        int nrKeys = XMLHandler.countNodes(keysnode, "key");
        int nrValues = XMLHandler.countNodes(valuesnode, "value");
        allocate(nrKeys, nrValues);
        for (int i = 0; i < nrKeys; i++) {
            Node keynode = XMLHandler.getSubNodeByNr(keysnode, "key", i);
            keyFields[i] = XMLHandler.getNodeValue(keynode);
        }
        for (int i = 0; i < nrValues; i++) {
            Node valuenode = XMLHandler.getSubNodeByNr(valuesnode, "value", i);
            valueFields[i] = XMLHandler.getNodeValue(valuenode);
        }
        flagField = XMLHandler.getTagValue(stepnode, "flag_field");
        List<StreamInterface> infoStreams = getStepIOMeta().getInfoStreams();
        StreamInterface referenceStream = infoStreams.get(0);
        StreamInterface compareStream = infoStreams.get(1);
        compareStream.setSubject(XMLHandler.getTagValue(stepnode, "compare"));
        referenceStream.setSubject(XMLHandler.getTagValue(stepnode, "reference"));
    } catch (Exception e) {
        throw new KettleXMLException(BaseMessages.getString(PKG, "MergeRowsMeta.Exception.UnableToLoadStepInfo"), e);
    }
}

Example 59 with StreamInterface

use of org.pentaho.di.trans.step.errorhandling.StreamInterface in project pentaho-kettle by pentaho.

the class MergeRowsMeta method check.

@Override
public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore) {
    CheckResult cr;
    List<StreamInterface> infoStreams = getStepIOMeta().getInfoStreams();
    StreamInterface referenceStream = infoStreams.get(0);
    StreamInterface compareStream = infoStreams.get(1);
    if (referenceStream.getStepname() != null && compareStream.getStepname() != null) {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.SourceStepsOK"), stepMeta);
        remarks.add(cr);
    } else if (referenceStream.getStepname() == null && compareStream.getStepname() == null) {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.SourceStepsMissing"), stepMeta);
        remarks.add(cr);
    } else {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.OneSourceStepMissing"), stepMeta);
        remarks.add(cr);
    }
    RowMetaInterface referenceRowMeta = null;
    RowMetaInterface compareRowMeta = null;
    try {
        referenceRowMeta = transMeta.getPrevStepFields(referenceStream.getStepname());
        compareRowMeta = transMeta.getPrevStepFields(compareStream.getStepname());
    } catch (KettleStepException kse) {
        new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.ErrorGettingPrevStepFields"), stepMeta);
    }
    if (referenceRowMeta != null && compareRowMeta != null) {
        boolean rowsMatch = false;
        try {
            MergeRows.checkInputLayoutValid(referenceRowMeta, compareRowMeta);
            rowsMatch = true;
        } catch (KettleRowException kre) {
            rowsMatch = false;
        }
        if (rowsMatch) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.RowDefinitionMatch"), stepMeta);
            remarks.add(cr);
        } else {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.RowDefinitionNotMatch"), stepMeta);
            remarks.add(cr);
        }
    }
}

Example 60 with StreamInterface

use of org.pentaho.di.trans.step.errorhandling.StreamInterface in project pentaho-kettle by pentaho.

the class MultiMergeJoin method init.

/**
 * @see StepInterface#init(org.pentaho.di.trans.step.StepMetaInterface , org.pentaho.di.trans.step.StepDataInterface)
 */
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (MultiMergeJoinMeta) smi;
    data = (MultiMergeJoinData) sdi;
    if (super.init(smi, sdi)) {
        StepIOMetaInterface stepIOMeta = meta.getStepIOMeta();
        String[] inputStepNames = meta.getInputSteps();
        String inputStepName;
        List<StreamInterface> infoStreams = stepIOMeta.getInfoStreams();
        StreamInterface stream;
        for (int i = 0; i < infoStreams.size(); i++) {
            inputStepName = inputStepNames[i];
            stream = infoStreams.get(i);
            if (stream.getStepMeta() == null) {
                logError(BaseMessages.getString(PKG, "MultiMergeJoin.Log.UnableToFindReferenceStream", inputStepName));
                return false;
            }
        }
        String joinType = meta.getJoinType();
        for (int i = 0; i < MultiMergeJoinMeta.join_types.length; ++i) {
            if (joinType.equalsIgnoreCase(MultiMergeJoinMeta.join_types[i])) {
                data.optional = MultiMergeJoinMeta.optionals[i];
                return true;
            }
        }
        logError(BaseMessages.getString(PKG, "MultiMergeJoin.Log.InvalidJoinType", meta.getJoinType()));
        return false;
    }
    return true;
}