Examples with KettleRowException org.pentaho.di.core.exception.KettleRowException used on opensource projects

Example 1 with KettleRowException

use of org.pentaho.di.core.exception.KettleRowException in project pentaho-kettle by pentaho.

the class TransMeta method checkSteps.

/**
 * Checks all the steps and fills a List of (CheckResult) remarks.
 *
 * @param remarks
 *          The remarks list to add to.
 * @param only_selected
 *          true to check only the selected steps, false for all steps
 * @param monitor
 *          a progress monitor listener to be updated as the SQL statements are generated
 */
public void checkSteps(List<CheckResultInterface> remarks, boolean only_selected, ProgressMonitorListener monitor, VariableSpace space, Repository repository, IMetaStore metaStore) {
    try {
        // Start with a clean slate...
        remarks.clear();
        Map<ValueMetaInterface, String> values = new Hashtable<>();
        String[] stepnames;
        StepMeta[] steps;
        List<StepMeta> selectedSteps = getSelectedSteps();
        if (!only_selected || selectedSteps.isEmpty()) {
            stepnames = getStepNames();
            steps = getStepsArray();
        } else {
            stepnames = getSelectedStepNames();
            steps = selectedSteps.toArray(new StepMeta[selectedSteps.size()]);
        }
        ExtensionPointHandler.callExtensionPoint(getLogChannel(), KettleExtensionPoint.BeforeCheckSteps.id, new CheckStepsExtension(remarks, space, this, steps, repository, metaStore));
        boolean stop_checking = false;
        if (monitor != null) {
            monitor.beginTask(BaseMessages.getString(PKG, "TransMeta.Monitor.VerifyingThisTransformationTask.Title"), steps.length + 2);
        }
        for (int i = 0; i < steps.length && !stop_checking; i++) {
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.VerifyingStepTask.Title", stepnames[i]));
            }
            StepMeta stepMeta = steps[i];
            int nrinfo = findNrInfoSteps(stepMeta);
            StepMeta[] infostep = null;
            if (nrinfo > 0) {
                infostep = getInfoStep(stepMeta);
            }
            RowMetaInterface info = null;
            if (infostep != null) {
                try {
                    info = getStepFields(infostep);
                } catch (KettleStepException kse) {
                    info = null;
                    CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.ErrorOccurredGettingStepInfoFields.Description", "" + stepMeta, Const.CR + kse.getMessage()), stepMeta);
                    remarks.add(cr);
                }
            }
            // The previous fields from non-informative steps:
            RowMetaInterface prev = null;
            try {
                prev = getPrevStepFields(stepMeta);
            } catch (KettleStepException kse) {
                CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.ErrorOccurredGettingInputFields.Description", "" + stepMeta, Const.CR + kse.getMessage()), stepMeta);
                remarks.add(cr);
                // This is a severe error: stop checking...
                // Otherwise we wind up checking time & time again because nothing gets put in the database
                // cache, the timeout of certain databases is very long... (Oracle)
                stop_checking = true;
            }
            if (isStepUsedInTransHops(stepMeta) || getSteps().size() == 1) {
                // Get the input & output steps!
                // Copy to arrays:
                String[] input = getPrevStepNames(stepMeta);
                String[] output = getNextStepNames(stepMeta);
                // Check step specific info...
                ExtensionPointHandler.callExtensionPoint(getLogChannel(), KettleExtensionPoint.BeforeCheckStep.id, new CheckStepsExtension(remarks, space, this, new StepMeta[] { stepMeta }, repository, metaStore));
                stepMeta.check(remarks, this, prev, input, output, info, space, repository, metaStore);
                ExtensionPointHandler.callExtensionPoint(getLogChannel(), KettleExtensionPoint.AfterCheckStep.id, new CheckStepsExtension(remarks, space, this, new StepMeta[] { stepMeta }, repository, metaStore));
                // See if illegal characters etc. were used in field-names...
                if (prev != null) {
                    for (int x = 0; x < prev.size(); x++) {
                        ValueMetaInterface v = prev.getValueMeta(x);
                        String name = v.getName();
                        if (name == null) {
                            values.put(v, BaseMessages.getString(PKG, "TransMeta.Value.CheckingFieldName.FieldNameIsEmpty.Description"));
                        } else if (name.indexOf(' ') >= 0) {
                            values.put(v, BaseMessages.getString(PKG, "TransMeta.Value.CheckingFieldName.FieldNameContainsSpaces.Description"));
                        } else {
                            char[] list = new char[] { '.', ',', '-', '/', '+', '*', '\'', '\t', '"', '|', '@', '(', ')', '{', '}', '!', '^' };
                            for (int c = 0; c < list.length; c++) {
                                if (name.indexOf(list[c]) >= 0) {
                                    values.put(v, BaseMessages.getString(PKG, "TransMeta.Value.CheckingFieldName.FieldNameContainsUnfriendlyCodes.Description", String.valueOf(list[c])));
                                }
                            }
                        }
                    }
                    // Check if 2 steps with the same name are entering the step...
                    if (prev.size() > 1) {
                        String[] fieldNames = prev.getFieldNames();
                        String[] sortedNames = Const.sortStrings(fieldNames);
                        String prevName = sortedNames[0];
                        for (int x = 1; x < sortedNames.length; x++) {
                            // Checking for doubles
                            if (prevName.equalsIgnoreCase(sortedNames[x])) {
                                // Give a warning!!
                                CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultWarning.HaveTheSameNameField.Description", prevName), stepMeta);
                                remarks.add(cr);
                            } else {
                                prevName = sortedNames[x];
                            }
                        }
                    }
                } else {
                    CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.CannotFindPreviousFields.Description") + stepMeta.getName(), stepMeta);
                    remarks.add(cr);
                }
            } else {
                CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_WARNING, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultWarning.StepIsNotUsed.Description"), stepMeta);
                remarks.add(cr);
            }
            // Also check for mixing rows...
            try {
                checkRowMixingStatically(stepMeta, null);
            } catch (KettleRowException e) {
                CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, e.getMessage(), stepMeta);
                remarks.add(cr);
            }
            if (monitor != null) {
                // progress bar...
                monitor.worked(1);
                if (monitor.isCanceled()) {
                    stop_checking = true;
                }
            }
        }
        // Also, check the logging table of the transformation...
        if (monitor == null || !monitor.isCanceled()) {
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.CheckingTheLoggingTableTask.Title"));
            }
            if (transLogTable.getDatabaseMeta() != null) {
                Database logdb = new Database(this, transLogTable.getDatabaseMeta());
                logdb.shareVariablesWith(this);
                try {
                    logdb.connect();
                    CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultOK.ConnectingWorks.Description"), null);
                    remarks.add(cr);
                    if (transLogTable.getTableName() != null) {
                        if (logdb.checkTableExists(transLogTable.getTableName())) {
                            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultOK.LoggingTableExists.Description", transLogTable.getTableName()), null);
                            remarks.add(cr);
                            RowMetaInterface fields = transLogTable.getLogRecord(LogStatus.START, null, null).getRowMeta();
                            String sql = logdb.getDDL(transLogTable.getTableName(), fields);
                            if (sql == null || sql.length() == 0) {
                                cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultOK.CorrectLayout.Description"), null);
                                remarks.add(cr);
                            } else {
                                cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.LoggingTableNeedsAdjustments.Description") + Const.CR + sql, null);
                                remarks.add(cr);
                            }
                        } else {
                            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.LoggingTableDoesNotExist.Description"), null);
                            remarks.add(cr);
                        }
                    } else {
                        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultError.LogTableNotSpecified.Description"), null);
                        remarks.add(cr);
                    }
                } catch (KettleDatabaseException dbe) {
                // Ignore errors
                } finally {
                    logdb.disconnect();
                }
            }
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.CheckingForDatabaseUnfriendlyCharactersInFieldNamesTask.Title"));
        }
        if (values.size() > 0) {
            for (ValueMetaInterface v : values.keySet()) {
                String message = values.get(v);
                CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_WARNING, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultWarning.Description", v.getName(), message, v.getOrigin()), findStep(v.getOrigin()));
                remarks.add(cr);
            }
        } else {
            CheckResult cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TransMeta.CheckResult.TypeResultOK.Description"), null);
            remarks.add(cr);
        }
        if (monitor != null) {
            monitor.worked(1);
        }
        ExtensionPointHandler.callExtensionPoint(getLogChannel(), KettleExtensionPoint.AfterCheckSteps.id, new CheckStepsExtension(remarks, space, this, steps, repository, metaStore));
    } catch (Exception e) {
        log.logError(Const.getStackTracker(e));
        throw new RuntimeException(e);
    }
}

Example 2 with KettleRowException

use of org.pentaho.di.core.exception.KettleRowException in project pentaho-kettle by pentaho.

the class BaseStep method safeModeChecking.

/**
 * Safe mode checking.
 *
 * @param referenceRowMeta the reference row meta
 * @param rowMeta          the row meta
 * @throws KettleRowException the kettle row exception
 */
public static void safeModeChecking(RowMetaInterface referenceRowMeta, RowMetaInterface rowMeta) throws KettleRowException {
    // 
    if (referenceRowMeta.size() != rowMeta.size()) {
        throw new KettleRowException(BaseMessages.getString(PKG, "BaseStep.SafeMode.Exception.VaryingSize", "" + referenceRowMeta.size(), "" + rowMeta.size(), rowMeta.toString()));
    } else {
        // Check field by field for the position of the names...
        for (int i = 0; i < referenceRowMeta.size(); i++) {
            ValueMetaInterface referenceValue = referenceRowMeta.getValueMeta(i);
            ValueMetaInterface compareValue = rowMeta.getValueMeta(i);
            if (!referenceValue.getName().equalsIgnoreCase(compareValue.getName())) {
                throw new KettleRowException(BaseMessages.getString(PKG, "BaseStep.SafeMode.Exception.MixingLayout", "" + (i + 1), referenceValue.getName() + " " + referenceValue.toStringMeta(), compareValue.getName() + " " + compareValue.toStringMeta()));
            }
            if (referenceValue.getType() != compareValue.getType()) {
                throw new KettleRowException(BaseMessages.getString(PKG, "BaseStep.SafeMode.Exception.MixingTypes", "" + (i + 1), referenceValue.getName() + " " + referenceValue.toStringMeta(), compareValue.getName() + " " + compareValue.toStringMeta()));
            }
            if (referenceValue.getStorageType() != compareValue.getStorageType()) {
                throw new KettleRowException(BaseMessages.getString(PKG, "BaseStep.SafeMode.Exception.MixingStorageTypes", "" + (i + 1), referenceValue.getName() + " " + referenceValue.toStringMeta(), compareValue.getName() + " " + compareValue.toStringMeta()));
            }
        }
    }
}

Example 3 with KettleRowException

use of org.pentaho.di.core.exception.KettleRowException in project pentaho-kettle by pentaho.

the class MergeRows method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MergeRowsMeta) smi;
    data = (MergeRowsData) sdi;
    if (first) {
        first = false;
        // Find the appropriate RowSet
        // 
        List<StreamInterface> infoStreams = meta.getStepIOMeta().getInfoStreams();
        // oneRowSet is the "Reference" stream
        data.oneRowSet = findInputRowSet(infoStreams.get(0).getStepname());
        // twoRowSet is the "Comparison" stream
        data.twoRowSet = findInputRowSet(infoStreams.get(1).getStepname());
        // rowSetWhenIdentical is use in case the comparison is IDENTICAL.
        // this should be the "Comparison" stream but can be the "Reference" stream for backward compatibility (PDI-736)
        String useRefWhenIdenticalVar = Const.NVL(System.getProperty(Const.KETTLE_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL), "N");
        if ("N".equalsIgnoreCase(useRefWhenIdenticalVar)) {
            // use the reference stream (as per documentation)
            useRefWhenIdentical = false;
        } else {
            // use the comparison stream (for backward compatibility)
            useRefWhenIdentical = true;
        }
        data.one = getRowFrom(data.oneRowSet);
        data.two = getRowFrom(data.twoRowSet);
        try {
            checkInputLayoutValid(data.oneRowSet.getRowMeta(), data.twoRowSet.getRowMeta());
        } catch (KettleRowException e) {
            throw new KettleException(BaseMessages.getString(PKG, "MergeRows.Exception.InvalidLayoutDetected"), e);
        }
        if (data.one != null) {
            // Find the key indexes:
            data.keyNrs = new int[meta.getKeyFields().length];
            for (int i = 0; i < data.keyNrs.length; i++) {
                data.keyNrs[i] = data.oneRowSet.getRowMeta().indexOfValue(meta.getKeyFields()[i]);
                if (data.keyNrs[i] < 0) {
                    String message = BaseMessages.getString(PKG, "MergeRows.Exception.UnableToFindFieldInReferenceStream", meta.getKeyFields()[i]);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
        }
        if (data.two != null) {
            data.valueNrs = new int[meta.getValueFields().length];
            for (int i = 0; i < data.valueNrs.length; i++) {
                data.valueNrs[i] = data.twoRowSet.getRowMeta().indexOfValue(meta.getValueFields()[i]);
                if (data.valueNrs[i] < 0) {
                    String message = BaseMessages.getString(PKG, "MergeRows.Exception.UnableToFindFieldInReferenceStream", meta.getValueFields()[i]);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
        }
    }
    if (log.isRowLevel()) {
        logRowlevel(BaseMessages.getString(PKG, "MergeRows.Log.DataInfo", data.one + "") + data.two);
    }
    if (data.one == null && data.two == null) {
        setOutputDone();
        return false;
    }
    if (data.outputRowMeta == null) {
        data.outputRowMeta = new RowMeta();
        if (data.one != null) {
            meta.getFields(data.outputRowMeta, getStepname(), new RowMetaInterface[] { data.oneRowSet.getRowMeta() }, null, this, repository, metaStore);
        } else {
            meta.getFields(data.outputRowMeta, getStepname(), new RowMetaInterface[] { data.twoRowSet.getRowMeta() }, null, this, repository, metaStore);
        }
    }
    Object[] outputRow;
    int outputIndex;
    String flagField = null;
    if (data.one == null && data.two != null) {
        // Record 2 is flagged as new!
        outputRow = data.two;
        outputIndex = data.twoRowSet.getRowMeta().size();
        flagField = VALUE_NEW;
        // Also get a next row from compare rowset...
        data.two = getRowFrom(data.twoRowSet);
    } else if (data.one != null && data.two == null) {
        // Record 1 is flagged as deleted!
        outputRow = data.one;
        outputIndex = data.oneRowSet.getRowMeta().size();
        flagField = VALUE_DELETED;
        // Also get a next row from reference rowset...
        data.one = getRowFrom(data.oneRowSet);
    } else {
        // OK, Here is the real start of the compare code!
        int compare = data.oneRowSet.getRowMeta().compare(data.one, data.two, data.keyNrs);
        if (compare == 0) {
            // The Key matches, we CAN compare the two rows...
            int compareValues = data.oneRowSet.getRowMeta().compare(data.one, data.two, data.valueNrs);
            if (compareValues == 0) {
                if (useRefWhenIdentical) {
                    // backwards compatible behavior: use the reference stream (PDI-736)
                    outputRow = data.one;
                    outputIndex = data.oneRowSet.getRowMeta().size();
                } else {
                    // documented behavior: use the comparison stream (PDI-736)
                    outputRow = data.two;
                    outputIndex = data.twoRowSet.getRowMeta().size();
                }
                flagField = VALUE_IDENTICAL;
            } else {
                // Return the compare (most recent) row
                // 
                outputRow = data.two;
                outputIndex = data.twoRowSet.getRowMeta().size();
                flagField = VALUE_CHANGED;
            }
            // Get a new row from both streams...
            data.one = getRowFrom(data.oneRowSet);
            data.two = getRowFrom(data.twoRowSet);
        } else {
            if (compare < 0) {
                // one < two
                outputRow = data.one;
                outputIndex = data.oneRowSet.getRowMeta().size();
                flagField = VALUE_DELETED;
                data.one = getRowFrom(data.oneRowSet);
            } else {
                outputRow = data.two;
                outputIndex = data.twoRowSet.getRowMeta().size();
                flagField = VALUE_NEW;
                data.two = getRowFrom(data.twoRowSet);
            }
        }
    }
    // send the row to the next steps...
    putRow(data.outputRowMeta, RowDataUtil.addValueData(outputRow, outputIndex, flagField));
    if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "MergeRows.LineNumber") + getLinesRead());
        }
    }
    return true;
}

Example 4 with KettleRowException

use of org.pentaho.di.core.exception.KettleRowException in project pentaho-kettle by pentaho.

the class MergeRowsMeta method check.

@Override
public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore) {
    CheckResult cr;
    List<StreamInterface> infoStreams = getStepIOMeta().getInfoStreams();
    StreamInterface referenceStream = infoStreams.get(0);
    StreamInterface compareStream = infoStreams.get(1);
    if (referenceStream.getStepname() != null && compareStream.getStepname() != null) {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.SourceStepsOK"), stepMeta);
        remarks.add(cr);
    } else if (referenceStream.getStepname() == null && compareStream.getStepname() == null) {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.SourceStepsMissing"), stepMeta);
        remarks.add(cr);
    } else {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.OneSourceStepMissing"), stepMeta);
        remarks.add(cr);
    }
    RowMetaInterface referenceRowMeta = null;
    RowMetaInterface compareRowMeta = null;
    try {
        referenceRowMeta = transMeta.getPrevStepFields(referenceStream.getStepname());
        compareRowMeta = transMeta.getPrevStepFields(compareStream.getStepname());
    } catch (KettleStepException kse) {
        new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.ErrorGettingPrevStepFields"), stepMeta);
    }
    if (referenceRowMeta != null && compareRowMeta != null) {
        boolean rowsMatch = false;
        try {
            MergeRows.checkInputLayoutValid(referenceRowMeta, compareRowMeta);
            rowsMatch = true;
        } catch (KettleRowException kre) {
            rowsMatch = false;
        }
        if (rowsMatch) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.RowDefinitionMatch"), stepMeta);
            remarks.add(cr);
        } else {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "MergeRowsMeta.CheckResult.RowDefinitionNotMatch"), stepMeta);
            remarks.add(cr);
        }
    }
}

Example 5 with KettleRowException

use of org.pentaho.di.core.exception.KettleRowException in project pentaho-kettle by pentaho.

the class Spoon method performNewTransHopChecks.

/**
 * @param transMeta transformation's meta
 * @param newHop hop to be checked
 * @return true when the hop was added, false if there was an error
 */
public boolean performNewTransHopChecks(TransMeta transMeta, TransHopMeta newHop) {
    boolean ok = true;
    if (transMeta.hasLoop(newHop.getToStep())) {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "TransGraph.Dialog.HopCausesLoop.Message"));
        mb.setText(BaseMessages.getString(PKG, "TransGraph.Dialog.HopCausesLoop.Title"));
        mb.open();
        ok = false;
    }
    if (ok) {
        // StackOverflow there ;-)
        try {
            if (!newHop.getToStep().getStepMetaInterface().excludeFromRowLayoutVerification()) {
                transMeta.checkRowMixingStatically(newHop.getToStep(), null);
            }
        } catch (KettleRowException re) {
            // Show warning about mixing rows with conflicting layouts...
            new ErrorDialog(shell, BaseMessages.getString(PKG, "TransGraph.Dialog.HopCausesRowMixing.Title"), BaseMessages.getString(PKG, "TransGraph.Dialog.HopCausesRowMixing.Message"), re);
        }
        verifyCopyDistribute(transMeta, newHop.getFromStep());
    }
    return ok;
}