Examples with KettleValueException org.pentaho.di.core.exception.KettleValueException used on opensource projects

Example 21 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class JavaFilter method calcFields.

private boolean calcFields(RowMetaInterface rowMeta, Object[] r) throws KettleValueException {
    try {
        // 
        if (data.expressionEvaluator == null) {
            String realCondition = environmentSubstitute(meta.getCondition());
            data.argumentIndexes = new ArrayList<Integer>();
            List<String> parameterNames = new ArrayList<String>();
            List<Class<?>> parameterTypes = new ArrayList<Class<?>>();
            for (int i = 0; i < data.outputRowMeta.size(); i++) {
                ValueMetaInterface valueMeta = data.outputRowMeta.getValueMeta(i);
                // 
                if (realCondition.contains(valueMeta.getName())) {
                    // If so, add it to the indexes...
                    data.argumentIndexes.add(i);
                    Class<?> parameterType;
                    switch(valueMeta.getType()) {
                        case ValueMetaInterface.TYPE_STRING:
                            parameterType = String.class;
                            break;
                        case ValueMetaInterface.TYPE_NUMBER:
                            parameterType = Double.class;
                            break;
                        case ValueMetaInterface.TYPE_INTEGER:
                            parameterType = Long.class;
                            break;
                        case ValueMetaInterface.TYPE_DATE:
                            parameterType = Date.class;
                            break;
                        case ValueMetaInterface.TYPE_BIGNUMBER:
                            parameterType = BigDecimal.class;
                            break;
                        case ValueMetaInterface.TYPE_BOOLEAN:
                            parameterType = Boolean.class;
                            break;
                        case ValueMetaInterface.TYPE_BINARY:
                            parameterType = byte[].class;
                            break;
                        default:
                            parameterType = String.class;
                            break;
                    }
                    parameterTypes.add(parameterType);
                    parameterNames.add(valueMeta.getName());
                }
            }
            // Create the expression evaluator: is relatively slow so we do it only for the first row...
            // 
            data.expressionEvaluator = new ExpressionEvaluator();
            data.expressionEvaluator.setParameters(parameterNames.toArray(new String[parameterNames.size()]), parameterTypes.toArray(new Class<?>[parameterTypes.size()]));
            data.expressionEvaluator.setReturnType(Object.class);
            data.expressionEvaluator.setThrownExceptions(new Class<?>[] { Exception.class });
            data.expressionEvaluator.cook(realCondition);
            // Also create the argument data structure once...
            // 
            data.argumentData = new Object[data.argumentIndexes.size()];
        }
        // 
        for (int x = 0; x < data.argumentIndexes.size(); x++) {
            int index = data.argumentIndexes.get(x);
            ValueMetaInterface outputValueMeta = data.outputRowMeta.getValueMeta(index);
            data.argumentData[x] = outputValueMeta.convertToNormalStorageType(r[index]);
        }
        Object formulaResult = data.expressionEvaluator.evaluate(data.argumentData);
        if (formulaResult instanceof Boolean) {
            return (Boolean) formulaResult;
        } else {
            throw new KettleException("The result of the filter expression must be a boolean and we got back : " + formulaResult.getClass().getName());
        }
    } catch (Exception e) {
        throw new KettleValueException(e);
    }
}

Example 22 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class ConcatFieldsMeta method getFields.

@Override
public void getFields(RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // remove selected fields from the stream when true
    if (removeSelectedFields) {
        if (getOutputFields().length > 0) {
            for (int i = 0; i < getOutputFields().length; i++) {
                TextFileField field = getOutputFields()[i];
                try {
                    row.removeValueMeta(field.getName());
                } catch (KettleValueException e) {
                // just ignore exceptions since missing fields are handled in the ConcatFields class
                }
            }
        } else {
            // no output fields selected, take them all, remove them all
            row.clear();
        }
    }
    // Check Target Field Name
    if (Utils.isEmpty(targetFieldName)) {
        throw new KettleStepException(BaseMessages.getString(PKG, "ConcatFieldsMeta.CheckResult.TargetFieldNameMissing"));
    }
    // add targetFieldName
    ValueMetaInterface vValue = new ValueMetaString(targetFieldName);
    vValue.setLength(targetFieldLength, 0);
    vValue.setOrigin(name);
    if (!Utils.isEmpty(getEncoding())) {
        vValue.setStringEncoding(getEncoding());
    }
    row.addValueMeta(vValue);
}

Example 23 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class CsvInput method readOneRow.

/**
 * Read a single row of data from the file...
 *
 * @param skipRow          if row should be skipped: header row or part of row in case of parallel read
 * @param ignoreEnclosures if enclosures should be ignored, i.e. in case of we need to skip part of the row during
 *                         parallel read
 * @return a row of data...
 * @throws KettleException
 */
private Object[] readOneRow(boolean skipRow, boolean ignoreEnclosures) throws KettleException {
    try {
        Object[] outputRowData = RowDataUtil.allocateRowData(data.outputRowMeta.size());
        int outputIndex = 0;
        boolean newLineFound = false;
        boolean endOfBuffer = false;
        List<Exception> conversionExceptions = null;
        List<ValueMetaInterface> exceptionFields = null;
        // 
        while (!newLineFound && outputIndex < data.fieldsMapping.size()) {
            if (data.resizeBufferIfNeeded()) {
                // there is no end of line delimiter
                if (outputRowData != null) {
                    // filling the rest of them with null
                    if (outputIndex > 0) {
                        // 
                        if (meta.isIncludingFilename() && !Utils.isEmpty(meta.getFilenameField())) {
                            if (meta.isLazyConversionActive()) {
                                outputRowData[data.filenameFieldIndex] = data.binaryFilename;
                            } else {
                                outputRowData[data.filenameFieldIndex] = data.filenames[data.filenr - 1];
                            }
                        }
                        if (data.isAddingRowNumber) {
                            outputRowData[data.rownumFieldIndex] = data.rowNumber++;
                        }
                        incrementLinesInput();
                        return outputRowData;
                    }
                }
                // nothing more to read, call it a day.
                return null;
            }
            // OK, at this point we should have data in the byteBuffer and we should be able to scan for the next
            // delimiter (;)
            // So let's look for a delimiter.
            // Also skip over the enclosures ("), it is NOT taking into account escaped enclosures.
            // Later we can add an option for having escaped or double enclosures in the file. <sigh>
            // 
            boolean delimiterFound = false;
            boolean enclosureFound = false;
            boolean doubleLineEnd = false;
            int escapedEnclosureFound = 0;
            boolean ignoreEnclosuresInField = ignoreEnclosures;
            while (!delimiterFound && !newLineFound && !endOfBuffer) {
                // 
                if (data.delimiterFound()) {
                    delimiterFound = true;
                } else if ((!meta.isNewlinePossibleInFields() || outputIndex == data.fieldsMapping.size() - 1) && data.newLineFound()) {
                    // Perhaps we found a (pre-mature) new line?
                    // 
                    // In case we are not using an enclosure and in case fields contain new lines
                    // we need to make sure that we check the newlines possible flag.
                    // If the flag is enable we skip newline checking except for the last field in the row.
                    // In that one we can't support newlines without enclosure (handled below).
                    // 
                    newLineFound = true;
                    // Skip new line character
                    for (int i = 0; i < data.encodingType.getLength(); i++) {
                        data.moveEndBufferPointer();
                    }
                    // Re-check for double new line (\r\n)...
                    if (data.newLineFound()) {
                        // Found another one, need to skip it later
                        doubleLineEnd = true;
                    }
                } else if (data.enclosureFound() && !ignoreEnclosuresInField) {
                    int enclosurePosition = data.getEndBuffer();
                    int fieldFirstBytePosition = data.getStartBuffer();
                    if (fieldFirstBytePosition == enclosurePosition) {
                        // Perhaps we need to skip over an enclosed part?
                        // We always expect exactly one enclosure character
                        // If we find the enclosure doubled, we consider it escaped.
                        // --> "" is converted to " later on.
                        // 
                        enclosureFound = true;
                        boolean keepGoing;
                        do {
                            if (data.moveEndBufferPointer()) {
                                enclosureFound = false;
                                break;
                            }
                            keepGoing = !data.enclosureFound();
                            if (!keepGoing) {
                                // Read another byte...
                                if (!data.endOfBuffer() && data.moveEndBufferPointer()) {
                                    break;
                                }
                                if (data.enclosure.length > 1) {
                                    data.moveEndBufferPointer();
                                }
                                // If this character is also an enclosure, we can consider the enclosure "escaped".
                                // As such, if this is an enclosure, we keep going...
                                // 
                                keepGoing = data.enclosureFound();
                                if (keepGoing) {
                                    escapedEnclosureFound++;
                                }
                            }
                        } while (keepGoing);
                        // 
                        if (data.endOfBuffer()) {
                            endOfBuffer = true;
                            break;
                        }
                    } else {
                        // Ignoring enclosure if it's not at the field start
                        ignoreEnclosuresInField = true;
                    }
                } else {
                    if (data.moveEndBufferPointer()) {
                        endOfBuffer = true;
                        break;
                    }
                }
            }
            // If we're still here, we found a delimiter...
            // Since the starting point never changed really, we just can grab range:
            // 
            // [startBuffer-endBuffer[
            // 
            // This is the part we want.
            // data.byteBuffer[data.startBuffer]
            // 
            byte[] field = data.getField(delimiterFound, enclosureFound, newLineFound, endOfBuffer);
            // 
            if (escapedEnclosureFound > 0) {
                if (log.isRowLevel()) {
                    logRowlevel("Escaped enclosures found in " + new String(field));
                }
                field = data.removeEscapedEnclosures(field, escapedEnclosureFound);
            }
            final int currentFieldIndex = outputIndex++;
            final int actualFieldIndex = data.fieldsMapping.fieldMetaIndex(currentFieldIndex);
            if (actualFieldIndex != FieldsMapping.FIELD_DOES_NOT_EXIST) {
                if (!skipRow) {
                    if (meta.isLazyConversionActive()) {
                        outputRowData[actualFieldIndex] = field;
                    } else {
                        // We're not lazy so we convert the data right here and now.
                        // The convert object uses binary storage as such we just have to ask the native type from it.
                        // That will do the actual conversion.
                        // 
                        ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(actualFieldIndex);
                        try {
                            outputRowData[actualFieldIndex] = sourceValueMeta.convertBinaryStringToNativeType(field);
                        } catch (KettleValueException e) {
                            // There was a conversion error,
                            // 
                            outputRowData[actualFieldIndex] = null;
                            if (conversionExceptions == null) {
                                conversionExceptions = new ArrayList<Exception>();
                                exceptionFields = new ArrayList<ValueMetaInterface>();
                            }
                            conversionExceptions.add(e);
                            exceptionFields.add(sourceValueMeta);
                        }
                    }
                } else {
                    // nothing for the header, no conversions here.
                    outputRowData[actualFieldIndex] = null;
                }
            }
            // empty column at the end of the row (see the Jira case for details)
            if ((!newLineFound && outputIndex < data.fieldsMapping.size()) || (newLineFound && doubleLineEnd)) {
                int i = 0;
                while ((!data.newLineFound() && (i < data.delimiter.length))) {
                    data.moveEndBufferPointer();
                    i++;
                }
                if (data.newLineFound() && outputIndex >= data.fieldsMapping.size()) {
                    data.moveEndBufferPointer();
                }
                if (doubleLineEnd && data.encodingType.getLength() > 1) {
                    data.moveEndBufferPointer();
                }
            }
            data.setStartBuffer(data.getEndBuffer());
        }
        // 
        if (!newLineFound && !data.resizeBufferIfNeeded()) {
            do {
                data.moveEndBufferPointer();
                if (data.resizeBufferIfNeeded()) {
                    // nothing more to read.
                    break;
                }
            // TODO: if we're using quoting we might be dealing with a very dirty file with quoted newlines in trailing
            // fields. (imagine that)
            // In that particular case we want to use the same logic we use above (refactored a bit) to skip these fields.
            } while (!data.newLineFound());
            if (!data.resizeBufferIfNeeded()) {
                while (data.newLineFound()) {
                    data.moveEndBufferPointer();
                    if (data.resizeBufferIfNeeded()) {
                        // nothing more to read.
                        break;
                    }
                }
            }
            // Make sure we start at the right position the next time around.
            data.setStartBuffer(data.getEndBuffer());
        }
        // 
        if (meta.isIncludingFilename() && !Utils.isEmpty(meta.getFilenameField())) {
            if (meta.isLazyConversionActive()) {
                outputRowData[data.filenameFieldIndex] = data.binaryFilename;
            } else {
                outputRowData[data.filenameFieldIndex] = data.filenames[data.filenr - 1];
            }
        }
        if (data.isAddingRowNumber) {
            outputRowData[data.rownumFieldIndex] = data.rowNumber++;
        }
        if (!ignoreEnclosures) {
            incrementLinesInput();
        }
        if (conversionExceptions != null && conversionExceptions.size() > 0) {
            // 
            throw new KettleConversionException("There were " + conversionExceptions.size() + " conversion errors on line " + getLinesInput(), conversionExceptions, exceptionFields, outputRowData);
        }
        return outputRowData;
    } catch (KettleConversionException e) {
        throw e;
    } catch (IOException e) {
        throw new KettleFileException("Exception reading line using NIO", e);
    }
}

Example 24 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class RowForumulaContext method resolveReference.

/**
 * We return the content of a Value with the given name. We cache the position of the field indexes.
 *
 * @see org.jfree.formula.FormulaContext#resolveReference(java.lang.Object)
 */
public Object resolveReference(Object name) throws EvaluationException {
    if (name instanceof String) {
        ValueMetaInterface valueMeta;
        Integer idx = valueIndexMap.get(name);
        if (idx != null) {
            valueMeta = rowMeta.getValueMeta(idx.intValue());
        } else {
            int index = rowMeta.indexOfValue((String) name);
            if (index < 0) {
                ErrorValue errorValue = new LibFormulaErrorValue(LibFormulaErrorValue.ERROR_INVALID_ARGUMENT);
                throw new EvaluationException(errorValue);
            }
            valueMeta = rowMeta.getValueMeta(index);
            idx = new Integer(index);
            valueIndexMap.put((String) name, idx);
        }
        Object valueData = rowData[idx];
        try {
            return getPrimitive(valueMeta, valueData);
        } catch (KettleValueException e) {
            throw new EvaluationException(LibFormulaErrorValue.ERROR_ARITHMETIC_VALUE);
        }
    }
    return null;
}

Example 25 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class ValueDataUtilTest method testNVL.

@Test
public void testNVL() {
    // Test Kettle number types
    assertEquals(Double.valueOf("1.0"), calculate("1", "", ValueMetaInterface.TYPE_NUMBER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(Double.valueOf("2.0"), calculate("", "2", ValueMetaInterface.TYPE_NUMBER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(Double.valueOf("10.0"), calculate("10", "20", ValueMetaInterface.TYPE_NUMBER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_NUMBER, CalculatorMetaFunction.CALC_NVL));
    // Test Kettle string types
    assertEquals("1", calculate("1", "", ValueMetaInterface.TYPE_STRING, CalculatorMetaFunction.CALC_NVL));
    assertEquals("2", calculate("", "2", ValueMetaInterface.TYPE_STRING, CalculatorMetaFunction.CALC_NVL));
    assertEquals("10", calculate("10", "20", ValueMetaInterface.TYPE_STRING, CalculatorMetaFunction.CALC_NVL));
    assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_STRING, CalculatorMetaFunction.CALC_NVL));
    // Test Kettle Integer (Java Long) types
    assertEquals(Long.valueOf("1"), calculate("1", "", ValueMetaInterface.TYPE_INTEGER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(Long.valueOf("2"), calculate("", "2", ValueMetaInterface.TYPE_INTEGER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(Long.valueOf("10"), calculate("10", "20", ValueMetaInterface.TYPE_INTEGER, CalculatorMetaFunction.CALC_NVL));
    assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_INTEGER, CalculatorMetaFunction.CALC_NVL));
    // Test Kettle big Number types
    assertEquals(0, new BigDecimal("1").compareTo((BigDecimal) calculate("1", "", ValueMetaInterface.TYPE_BIGNUMBER, CalculatorMetaFunction.CALC_NVL)));
    assertEquals(0, new BigDecimal("2").compareTo((BigDecimal) calculate("", "2", ValueMetaInterface.TYPE_BIGNUMBER, CalculatorMetaFunction.CALC_NVL)));
    assertEquals(0, new BigDecimal("10").compareTo((BigDecimal) calculate("10", "20", ValueMetaInterface.TYPE_BIGNUMBER, CalculatorMetaFunction.CALC_NVL)));
    assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_BIGNUMBER, CalculatorMetaFunction.CALC_NVL));
    // boolean
    assertEquals(true, calculate("true", "", ValueMetaInterface.TYPE_BOOLEAN, CalculatorMetaFunction.CALC_NVL));
    assertEquals(false, calculate("", "false", ValueMetaInterface.TYPE_BOOLEAN, CalculatorMetaFunction.CALC_NVL));
    assertEquals(false, calculate("false", "true", ValueMetaInterface.TYPE_BOOLEAN, CalculatorMetaFunction.CALC_NVL));
    assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_BOOLEAN, CalculatorMetaFunction.CALC_NVL));
    // Test Kettle date
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat(yyyy_MM_dd);
    try {
        assertEquals(simpleDateFormat.parse("2012-04-11"), calculate("2012-04-11", "", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL));
        assertEquals(simpleDateFormat.parse("2012-11-04"), calculate("", "2012-11-04", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL));
        assertEquals(simpleDateFormat.parse("1965-07-01"), calculate("1965-07-01", "1967-04-11", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL));
        assertNull(calculate("", "", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL));
    } catch (ParseException pe) {
        fail(pe.getMessage());
    }
    // assertEquals(0, calculate("", "2012-11-04", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL)));
    // assertEquals(0, calculate("2012-11-04", "2010-04-11", ValueMetaInterface.TYPE_DATE,
    // CalculatorMetaFunction.CALC_NVL)));
    // assertEquals(null, calculate("", "", ValueMetaInterface.TYPE_DATE, CalculatorMetaFunction.CALC_NVL));
    // binary
    ValueMetaInterface stringValueMeta = new ValueMetaString("string");
    try {
        byte[] data = stringValueMeta.getBinary("101");
        byte[] calculated = (byte[]) calculate("101", "", ValueMetaInterface.TYPE_BINARY, CalculatorMetaFunction.CALC_NVL);
        assertTrue(Arrays.equals(data, calculated));
        data = stringValueMeta.getBinary("011");
        calculated = (byte[]) calculate("", "011", ValueMetaInterface.TYPE_BINARY, CalculatorMetaFunction.CALC_NVL);
        assertTrue(Arrays.equals(data, calculated));
        data = stringValueMeta.getBinary("110");
        calculated = (byte[]) calculate("110", "011", ValueMetaInterface.TYPE_BINARY, CalculatorMetaFunction.CALC_NVL);
        assertTrue(Arrays.equals(data, calculated));
        calculated = (byte[]) calculate("", "", ValueMetaInterface.TYPE_BINARY, CalculatorMetaFunction.CALC_NVL);
        assertNull(calculated);
    // assertEquals(binaryValueMeta.convertData(new ValueMeta("dummy", ValueMeta.TYPE_STRING), "101"),
    // calculate("101", "", ValueMetaInterface.TYPE_BINARY, CalculatorMetaFunction.CALC_NVL));
    } catch (KettleValueException kve) {
        fail(kve.getMessage());
    }
}