Examples with KettleValueException org.pentaho.di.core.exception.KettleValueException used on opensource projects

Example 46 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class StreamLookup method addToCache.

private void addToCache(RowMetaInterface keyMeta, Object[] keyData, RowMetaInterface valueMeta, Object[] valueData) throws KettleValueException {
    if (meta.isMemoryPreservationActive()) {
        if (meta.isUsingSortedList()) {
            KeyValue keyValue = new KeyValue(keyData, valueData);
            int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
            if (idx < 0) {
                // this is the insertion point
                int index = -idx - 1;
                // insert to keep sorted.
                data.list.add(index, keyValue);
            } else {
                // Overwrite to simulate Hashtable behaviour
                data.list.set(idx, keyValue);
            }
        } else {
            if (meta.isUsingIntegerPair()) {
                if (!data.metadataVerifiedIntegerPair) {
                    data.metadataVerifiedIntegerPair = true;
                    if (keyMeta.size() != 1 || valueMeta.size() != 1 || !keyMeta.getValueMeta(0).isInteger() || !valueMeta.getValueMeta(0).isInteger()) {
                        throw new KettleValueException(BaseMessages.getString(PKG, "StreamLookup.Exception.CanNotUseIntegerPairAlgorithm"));
                    }
                }
                Long key = keyMeta.getInteger(keyData, 0);
                Long value = valueMeta.getInteger(valueData, 0);
                data.longIndex.put(key, value);
            } else {
                if (data.hashIndex == null) {
                    data.hashIndex = new ByteArrayHashIndex(keyMeta);
                }
                data.hashIndex.put(RowMeta.extractData(keyMeta, keyData), RowMeta.extractData(valueMeta, valueData));
            }
        }
    } else {
        // We can't just put Object[] in the map The compare function is not in it.
        // We need to wrap in and use that. Let's use RowMetaAndData for this one.
        data.look.put(new RowMetaAndData(keyMeta, keyData), valueData);
    }
}

Example 47 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class StreamLookup method lookupValues.

private Object[] lookupValues(RowMetaInterface rowMeta, Object[] row) throws KettleException {
    // See if we need to stop.
    if (isStopped()) {
        return null;
    }
    if (data.lookupColumnIndex == null) {
        String[] names = data.lookupMeta.getFieldNames();
        data.lookupColumnIndex = new int[names.length];
        for (int i = 0; i < names.length; i++) {
            data.lookupColumnIndex[i] = rowMeta.indexOfValue(names[i]);
            if (data.lookupColumnIndex[i] < 0) {
                // we should not get here
                throw new KettleStepException("The lookup column '" + names[i] + "' could not be found");
            }
        }
    }
    // Copy value references to lookup table.
    // 
    Object[] lu = new Object[data.keynrs.length];
    for (int i = 0; i < data.keynrs.length; i++) {
        // 
        if (data.convertKeysToNative[i]) {
            lu[i] = data.lookupMeta.getValueMeta(i).convertBinaryStringToNativeType((byte[]) row[data.keynrs[i]]);
        } else {
            lu[i] = row[data.keynrs[i]];
        }
    }
    // Handle conflicting types (Number-Integer-String conversion to lookup type in hashtable)
    if (data.keyTypes != null) {
        for (int i = 0; i < data.lookupMeta.size(); i++) {
            ValueMetaInterface inputValue = data.lookupMeta.getValueMeta(i);
            ValueMetaInterface lookupValue = data.keyTypes.getValueMeta(i);
            if (inputValue.getType() != lookupValue.getType()) {
                try {
                    // Change the input value to match the lookup value
                    // 
                    lu[i] = lookupValue.convertDataCompatible(inputValue, lu[i]);
                } catch (KettleValueException e) {
                    throw new KettleStepException("Error converting data while looking up value", e);
                }
            }
        }
    }
    Object[] add = null;
    if (data.hasLookupRows) {
        try {
            if (meta.getKeystream().length > 0) {
                add = getFromCache(data.cacheKeyMeta, lu);
            } else {
                // Just take the first element in the hashtable...
                throw new KettleStepException(BaseMessages.getString(PKG, "StreamLookup.Log.GotRowWithoutKeys"));
            }
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
    if (add == null) {
        // nothing was found, unknown code: add the specified default value...
        add = data.nullIf;
    }
    return RowDataUtil.addRowData(row, rowMeta.size(), add);
}

Example 48 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class RowGenerator method buildRow.

public static final RowMetaAndData buildRow(RowGeneratorMeta meta, List<CheckResultInterface> remarks, String origin) throws KettlePluginException {
    RowMetaInterface rowMeta = new RowMeta();
    Object[] rowData = RowDataUtil.allocateRowData(meta.getFieldName().length + 2);
    int index = 0;
    if (meta.isNeverEnding()) {
        if (!Utils.isEmpty(meta.getRowTimeField())) {
            rowMeta.addValueMeta(new ValueMetaDate(meta.getRowTimeField()));
            rowData[index++] = null;
        }
        if (!Utils.isEmpty(meta.getLastTimeField())) {
            rowMeta.addValueMeta(new ValueMetaDate(meta.getLastTimeField()));
            rowData[index++] = null;
        }
    }
    for (int i = 0; i < meta.getFieldName().length; i++) {
        int valtype = ValueMetaFactory.getIdForValueMeta(meta.getFieldType()[i]);
        if (meta.getFieldName()[i] != null) {
            // build a
            ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(meta.getFieldName()[i], valtype);
            // value!
            valueMeta.setLength(meta.getFieldLength()[i]);
            valueMeta.setPrecision(meta.getFieldPrecision()[i]);
            valueMeta.setConversionMask(meta.getFieldFormat()[i]);
            valueMeta.setCurrencySymbol(meta.getCurrency()[i]);
            valueMeta.setGroupingSymbol(meta.getGroup()[i]);
            valueMeta.setDecimalSymbol(meta.getDecimal()[i]);
            valueMeta.setOrigin(origin);
            ValueMetaInterface stringMeta = ValueMetaFactory.cloneValueMeta(valueMeta, ValueMetaInterface.TYPE_STRING);
            if (meta.isSetEmptyString() != null && meta.isSetEmptyString()[i]) {
                // Set empty string
                rowData[index] = StringUtil.EMPTY_STRING;
            } else {
                String stringValue = meta.getValue()[i];
                // If the value is empty: consider it to be NULL.
                if (Utils.isEmpty(stringValue)) {
                    rowData[index] = null;
                    if (valueMeta.getType() == ValueMetaInterface.TYPE_NONE) {
                        String message = BaseMessages.getString(PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue);
                        remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                    }
                } else {
                    // 
                    try {
                        rowData[index] = valueMeta.convertData(stringMeta, stringValue);
                    } catch (KettleValueException e) {
                        switch(valueMeta.getType()) {
                            case ValueMetaInterface.TYPE_NUMBER:
                                String message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Number", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                            case ValueMetaInterface.TYPE_DATE:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Date", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                            case ValueMetaInterface.TYPE_INTEGER:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Integer", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                            case ValueMetaInterface.TYPE_BIGNUMBER:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.BigNumber", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                            case ValueMetaInterface.TYPE_TIMESTAMP:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Timestamp", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                            default:
                                // Boolean and binary don't throw errors normally, so it's probably an unspecified error problem...
                                message = BaseMessages.getString(PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue);
                                remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
                                break;
                        }
                    }
                }
            }
            // Now add value to the row!
            // This is in fact a copy from the fields row, but now with data.
            rowMeta.addValueMeta(valueMeta);
            index++;
        }
    }
    return new RowMetaAndData(rowMeta, rowData);
}

Example 49 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class ScriptValuesMod method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (ScriptValuesMetaMod) smi;
    data = (ScriptValuesModData) sdi;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // Modification for Additional End Function
        try {
            if (data.cx != null) {
                // Checking for EndScript
                if (strEndScript != null && strEndScript.length() > 0) {
                    Script endScript = data.cx.compileString(strEndScript, "trans_End", 1, null);
                    endScript.exec(data.cx, data.scope);
                    if (log.isDetailed()) {
                        logDetailed(("End Script found!"));
                    }
                } else {
                    if (log.isDetailed()) {
                        logDetailed(("No end Script found!"));
                    }
                }
            }
        } catch (Exception e) {
            logError(BaseMessages.getString(PKG, "ScriptValuesMod.Log.UnexpectedeError") + " : " + e.toString());
            logError(BaseMessages.getString(PKG, "ScriptValuesMod.Log.ErrorStackTrace") + Const.CR + Const.getStackTracker(e));
            setErrors(1);
            stopAll();
        }
        try {
            if (data.cx != null) {
                Context.exit();
            }
        } catch (Exception er) {
        // Eat this error, it's typically : "Calling Context.exit without previous Context.enter"
        // logError(BaseMessages.getString(PKG, "System.Log.UnexpectedError"), er);
        }
        setOutputDone();
        return false;
    }
    // Getting the Row, with the Transformation Status
    try {
        addValues(getInputRowMeta(), r);
    } catch (KettleValueException e) {
        String location = null;
        if (e.getCause() instanceof EvaluatorException) {
            EvaluatorException ee = (EvaluatorException) e.getCause();
            location = "--> " + ee.lineNumber() + ":" + ee.columnNumber();
        }
        if (getStepMeta().isDoingErrorHandling()) {
            putError(getInputRowMeta(), r, 1, e.getMessage() + Const.CR + location, null, "SCR-001");
            // continue by all means, even on the first row and out of this ugly design
            bRC = true;
        } else {
            throw (e);
        }
    }
    if (checkFeedback(getLinesRead())) {
        logBasic(BaseMessages.getString(PKG, "ScriptValuesMod.Log.LineNumber") + getLinesRead());
    }
    return bRC;
}

Example 50 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class ScriptValuesMod method addValues.

private boolean addValues(RowMetaInterface rowMeta, Object[] row) throws KettleException {
    if (first) {
        first = false;
        // What is the output row looking like?
        // 
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Determine the indexes of the fields used!
        // 
        determineUsedFields(rowMeta);
        // Get the indexes of the replaced fields...
        // 
        data.replaceIndex = new int[meta.getFieldname().length];
        for (int i = 0; i < meta.getFieldname().length; i++) {
            if (meta.getReplace()[i]) {
                data.replaceIndex[i] = rowMeta.indexOfValue(meta.getFieldname()[i]);
                if (data.replaceIndex[i] < 0) {
                    if (Utils.isEmpty(meta.getFieldname()[i])) {
                        throw new KettleStepException(BaseMessages.getString(PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", meta.getFieldname()[i]));
                    }
                    data.replaceIndex[i] = rowMeta.indexOfValue(meta.getRename()[i]);
                    if (data.replaceIndex[i] < 0) {
                        throw new KettleStepException(BaseMessages.getString(PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", meta.getRename()[i]));
                    }
                }
            } else {
                data.replaceIndex[i] = -1;
            }
        }
        // set the optimization level
        data.cx = ContextFactory.getGlobal().enterContext();
        try {
            String optimizationLevelAsString = environmentSubstitute(meta.getOptimizationLevel());
            if (!Utils.isEmpty(Const.trim(optimizationLevelAsString))) {
                data.cx.setOptimizationLevel(Integer.parseInt(optimizationLevelAsString.trim()));
                logBasic(BaseMessages.getString(PKG, "ScriptValuesMod.Optimization.Level", environmentSubstitute(meta.getOptimizationLevel())));
            } else {
                data.cx.setOptimizationLevel(Integer.parseInt(ScriptValuesMetaMod.OPTIMIZATION_LEVEL_DEFAULT));
                logBasic(BaseMessages.getString(PKG, "ScriptValuesMod.Optimization.UsingDefault", ScriptValuesMetaMod.OPTIMIZATION_LEVEL_DEFAULT));
            }
        } catch (NumberFormatException nfe) {
            throw new KettleStepException(BaseMessages.getString(PKG, "ScriptValuesMetaMod.Exception.NumberFormatException", environmentSubstitute(meta.getOptimizationLevel())));
        } catch (IllegalArgumentException iae) {
            throw new KettleException(iae.getMessage());
        }
        data.scope = data.cx.initStandardObjects(null, false);
        bFirstRun = true;
        Scriptable jsvalue = Context.toObject(this, data.scope);
        data.scope.put("_step_", data.scope, jsvalue);
        // Adding the existing Scripts to the Context
        for (int i = 0; i < meta.getNumberOfJSScripts(); i++) {
            Scriptable jsR = Context.toObject(jsScripts[i].getScript(), data.scope);
            data.scope.put(jsScripts[i].getScriptName(), data.scope, jsR);
        }
        // Adding the Name of the Transformation to the Context
        data.scope.put("_TransformationName_", data.scope, getTransMeta().getName());
        try {
            // 
            if (meta.isCompatible()) {
                Row v2Row = RowMeta.createOriginalRow(rowMeta, row);
                Scriptable jsV2Row = Context.toObject(v2Row, data.scope);
                data.scope.put("row", data.scope, jsV2Row);
            } else {
                Scriptable jsrow = Context.toObject(row, data.scope);
                data.scope.put("row", data.scope, jsrow);
            }
            // 
            for (int i = 0; i < data.fields_used.length; i++) {
                ValueMetaInterface valueMeta = rowMeta.getValueMeta(data.fields_used[i]);
                Object valueData = row[data.fields_used[i]];
                if (meta.isCompatible()) {
                    data.values_used[i] = valueMeta.createOriginalValue(valueData);
                    Scriptable jsarg = Context.toObject(data.values_used[i], data.scope);
                    data.scope.put(valueMeta.getName(), data.scope, jsarg);
                } else {
                    Object normalStorageValueData = valueMeta.convertToNormalStorageType(valueData);
                    Scriptable jsarg;
                    if (normalStorageValueData != null) {
                        jsarg = Context.toObject(normalStorageValueData, data.scope);
                    } else {
                        jsarg = null;
                    }
                    data.scope.put(valueMeta.getName(), data.scope, jsarg);
                }
            }
            // also add the meta information for the whole row
            // 
            Scriptable jsrowMeta = Context.toObject(rowMeta, data.scope);
            data.scope.put("rowMeta", data.scope, jsrowMeta);
            // 
            try {
                if (meta.getAddClasses() != null) {
                    for (int i = 0; i < meta.getAddClasses().length; i++) {
                        Object jsOut = Context.javaToJS(meta.getAddClasses()[i].getAddObject(), data.scope);
                        ScriptableObject.putProperty(data.scope, meta.getAddClasses()[i].getJSName(), jsOut);
                    }
                }
            } catch (Exception e) {
                throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.CouldNotAttachAdditionalScripts"), e);
            }
            // Adding some default JavaScriptFunctions to the System
            try {
                Context.javaToJS(ScriptValuesAddedFunctions.class, data.scope);
                ((ScriptableObject) data.scope).defineFunctionProperties(ScriptValuesAddedFunctions.jsFunctionList, ScriptValuesAddedFunctions.class, ScriptableObject.DONTENUM);
            } catch (Exception ex) {
                // System.out.println(ex.toString());
                throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.CouldNotAddDefaultFunctions"), ex);
            }
            // Adding some Constants to the JavaScript
            try {
                data.scope.put("SKIP_TRANSFORMATION", data.scope, Integer.valueOf(SKIP_TRANSFORMATION));
                data.scope.put("ABORT_TRANSFORMATION", data.scope, Integer.valueOf(ABORT_TRANSFORMATION));
                data.scope.put("ERROR_TRANSFORMATION", data.scope, Integer.valueOf(ERROR_TRANSFORMATION));
                data.scope.put("CONTINUE_TRANSFORMATION", data.scope, Integer.valueOf(CONTINUE_TRANSFORMATION));
            } catch (Exception ex) {
                // System.out.println("Exception Adding the Constants " + ex.toString());
                throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.CouldNotAddDefaultConstants"), ex);
            }
            try {
                // Checking for StartScript
                if (strStartScript != null && strStartScript.length() > 0) {
                    Script startScript = data.cx.compileString(strStartScript, "trans_Start", 1, null);
                    startScript.exec(data.cx, data.scope);
                    if (log.isDetailed()) {
                        logDetailed(("Start Script found!"));
                    }
                } else {
                    if (log.isDetailed()) {
                        logDetailed(("No starting Script found!"));
                    }
                }
            } catch (Exception es) {
                // System.out.println("Exception processing StartScript " + es.toString());
                throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.ErrorProcessingStartScript"), es);
            }
            // Now Compile our Script
            data.script = data.cx.compileString(strTransformScript, "script", 1, null);
        } catch (Exception e) {
            throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.CouldNotCompileJavascript"), e);
        }
    }
    // Filling the defined TranVars with the Values from the Row
    // 
    Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
    // Keep an index...
    int outputIndex = rowMeta.size();
    // Keep track of the changed values...
    // 
    final Map<Integer, Value> usedRowValues;
    if (meta.isCompatible()) {
        usedRowValues = new Hashtable<Integer, Value>();
    } else {
        usedRowValues = null;
    }
    try {
        try {
            if (meta.isCompatible()) {
                Row v2Row = RowMeta.createOriginalRow(rowMeta, row);
                Scriptable jsV2Row = Context.toObject(v2Row, data.scope);
                data.scope.put("row", data.scope, jsV2Row);
                v2Row.getUsedValueListeners().add(new ValueUsedListener() {

                    public void valueIsUsed(int index, Value value) {
                        usedRowValues.put(index, value);
                    }
                });
            } else {
                Scriptable jsrow = Context.toObject(row, data.scope);
                data.scope.put("row", data.scope, jsrow);
            }
            for (int i = 0; i < data.fields_used.length; i++) {
                ValueMetaInterface valueMeta = rowMeta.getValueMeta(data.fields_used[i]);
                Object valueData = row[data.fields_used[i]];
                if (meta.isCompatible()) {
                    data.values_used[i] = valueMeta.createOriginalValue(valueData);
                    Scriptable jsarg = Context.toObject(data.values_used[i], data.scope);
                    data.scope.put(valueMeta.getName(), data.scope, jsarg);
                } else {
                    Object normalStorageValueData = valueMeta.convertToNormalStorageType(valueData);
                    Scriptable jsarg;
                    if (normalStorageValueData != null) {
                        jsarg = Context.toObject(normalStorageValueData, data.scope);
                    } else {
                        jsarg = null;
                    }
                    data.scope.put(valueMeta.getName(), data.scope, jsarg);
                }
            }
            // also add the meta information for the hole row
            Scriptable jsrowMeta = Context.toObject(rowMeta, data.scope);
            data.scope.put("rowMeta", data.scope, jsrowMeta);
        } catch (Exception e) {
            throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.UnexpectedeError"), e);
        }
        // Executing our Script
        data.script.exec(data.cx, data.scope);
        if (bFirstRun) {
            bFirstRun = false;
            // Check if we had a Transformation Status
            Object tran_stat = data.scope.get("trans_Status", data.scope);
            if (tran_stat != ScriptableObject.NOT_FOUND) {
                bWithTransStat = true;
                if (log.isDetailed()) {
                    logDetailed(("tran_Status found. Checking transformation status while script execution."));
                }
            } else {
                if (log.isDetailed()) {
                    logDetailed(("No tran_Status found. Transformation status checking not available."));
                }
                bWithTransStat = false;
            }
        }
        if (bWithTransStat) {
            iTranStat = (int) Context.toNumber(data.scope.get("trans_Status", data.scope));
        } else {
            iTranStat = CONTINUE_TRANSFORMATION;
        }
        if (iTranStat == CONTINUE_TRANSFORMATION) {
            bRC = true;
            for (int i = 0; i < meta.getFieldname().length; i++) {
                Object result = data.scope.get(meta.getFieldname()[i], data.scope);
                Object valueData = getValueFromJScript(result, i);
                if (data.replaceIndex[i] < 0) {
                    outputRow[outputIndex++] = valueData;
                } else {
                    outputRow[data.replaceIndex[i]] = valueData;
                }
            }
            // 
            if (meta.isCompatible()) {
                for (int i = 0; i < data.values_used.length; i++) {
                    ValueMetaInterface valueMeta = rowMeta.getValueMeta(data.fields_used[i]);
                    outputRow[data.fields_used[i]] = valueMeta.getValueData(data.values_used[i]);
                }
                // 
                for (Integer index : usedRowValues.keySet()) {
                    Value value = usedRowValues.get(index);
                    ValueMetaInterface valueMeta = rowMeta.getValueMeta(index);
                    outputRow[index] = valueMeta.getValueData(value);
                }
            }
            putRow(data.outputRowMeta, outputRow);
        } else {
            switch(iTranStat) {
                case SKIP_TRANSFORMATION:
                    // eat this row.
                    bRC = true;
                    break;
                case ABORT_TRANSFORMATION:
                    if (data.cx != null) {
                        Context.exit();
                    }
                    stopAll();
                    setOutputDone();
                    bRC = false;
                    break;
                case ERROR_TRANSFORMATION:
                    if (data.cx != null) {
                        Context.exit();
                    }
                    setErrors(1);
                    stopAll();
                    bRC = false;
                    break;
                default:
                    break;
            }
        // TODO: kick this "ERROR handling" junk out now that we have solid error handling in place.
        // 
        }
    } catch (Exception e) {
        throw new KettleValueException(BaseMessages.getString(PKG, "ScriptValuesMod.Log.JavascriptError"), e);
    }
    return bRC;
}