Examples with ValueMetaNumber org.pentaho.di.core.row.value.ValueMetaNumber used on opensource projects

Example 16 with ValueMetaNumber

use of org.pentaho.di.core.row.value.ValueMetaNumber in project pentaho-kettle by pentaho.

the class FuzzyMatchMeta method getFields.

public void getFields(RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // Add match field
    ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getOutputMatchField()));
    v.setOrigin(name);
    v.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
    inputRowMeta.addValueMeta(v);
    String mainField = space.environmentSubstitute(getOutputValueField());
    if (!Utils.isEmpty(mainField) && isGetCloserValue()) {
        switch(getAlgorithmType()) {
            case FuzzyMatchMeta.OPERATION_TYPE_DAMERAU_LEVENSHTEIN:
            case FuzzyMatchMeta.OPERATION_TYPE_LEVENSHTEIN:
                v = new ValueMetaInteger(mainField);
                v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH);
                break;
            case FuzzyMatchMeta.OPERATION_TYPE_JARO:
            case FuzzyMatchMeta.OPERATION_TYPE_JARO_WINKLER:
            case FuzzyMatchMeta.OPERATION_TYPE_PAIR_SIMILARITY:
                v = new ValueMetaNumber(mainField);
                break;
            default:
                // Phonetic algorithms
                v = new ValueMetaString(mainField);
                break;
        }
        v.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
        v.setOrigin(name);
        inputRowMeta.addValueMeta(v);
    }
    boolean activateAdditionalFields = isGetCloserValue() || (getAlgorithmType() == FuzzyMatchMeta.OPERATION_TYPE_DOUBLE_METAPHONE) || (getAlgorithmType() == FuzzyMatchMeta.OPERATION_TYPE_SOUNDEX) || (getAlgorithmType() == FuzzyMatchMeta.OPERATION_TYPE_REFINED_SOUNDEX) || (getAlgorithmType() == FuzzyMatchMeta.OPERATION_TYPE_METAPHONE);
    if (activateAdditionalFields) {
        if (info != null && info.length == 1 && info[0] != null) {
            for (int i = 0; i < valueName.length; i++) {
                v = info[0].searchValueMeta(value[i]);
                if (v != null) {
                    // Configuration error/missing resources...
                    v.setName(valueName[i]);
                    v.setOrigin(name);
                    // Only normal storage goes into the cache
                    v.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
                    inputRowMeta.addValueMeta(v);
                } else {
                    throw new KettleStepException(BaseMessages.getString(PKG, "FuzzyMatchMeta.Exception.ReturnValueCanNotBeFound", value[i]));
                }
            }
        } else {
            for (int i = 0; i < valueName.length; i++) {
                v = new ValueMetaString(valueName[i]);
                v.setOrigin(name);
                inputRowMeta.addValueMeta(v);
            }
        }
    }
}

Example 17 with ValueMetaNumber

use of org.pentaho.di.core.row.value.ValueMetaNumber in project pentaho-kettle by pentaho.

the class GroupBy method newAggregate.

/**
 * used for junits in GroupByAggregationNullsTest
 *
 * @param r
 */
void newAggregate(Object[] r) {
    // Put all the counters at 0
    for (int i = 0; i < data.counts.length; i++) {
        data.counts[i] = 0;
    }
    data.distinctObjs = null;
    data.agg = new Object[data.subjectnrs.length];
    // sets all doubles to 0.0
    data.mean = new double[data.subjectnrs.length];
    data.aggMeta = new RowMeta();
    for (int i = 0; i < data.subjectnrs.length; i++) {
        ValueMetaInterface subjMeta = data.inputRowMeta.getValueMeta(data.subjectnrs[i]);
        Object v = null;
        ValueMetaInterface vMeta = null;
        int aggType = meta.getAggregateType()[i];
        switch(aggType) {
            case GroupByMeta.TYPE_GROUP_SUM:
            case GroupByMeta.TYPE_GROUP_AVERAGE:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE:
                if (subjMeta.isNumeric()) {
                    try {
                        vMeta = ValueMetaFactory.createValueMeta(meta.getAggregateField()[i], subjMeta.getType());
                    } catch (KettlePluginException e) {
                        vMeta = new ValueMetaNone(meta.getAggregateField()[i]);
                    }
                } else {
                    vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                }
                break;
            case GroupByMeta.TYPE_GROUP_MEDIAN:
            case GroupByMeta.TYPE_GROUP_PERCENTILE:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                v = new ArrayList<Double>();
                break;
            case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_COUNT_DISTINCT:
            case GroupByMeta.TYPE_GROUP_COUNT_ANY:
            case GroupByMeta.TYPE_GROUP_COUNT_ALL:
                vMeta = new ValueMetaInteger(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_FIRST:
            case GroupByMeta.TYPE_GROUP_LAST:
            case GroupByMeta.TYPE_GROUP_FIRST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_LAST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_MIN:
            case GroupByMeta.TYPE_GROUP_MAX:
                vMeta = subjMeta.clone();
                vMeta.setName(meta.getAggregateField()[i]);
                v = r == null ? null : r[data.subjectnrs[i]];
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_COMMA:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_STRING:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            default:
                // TODO raise an error here because we cannot continue successfully maybe the UI should validate this
                break;
        }
        if ((subjMeta != null) && (aggType != GroupByMeta.TYPE_GROUP_COUNT_ALL && aggType != GroupByMeta.TYPE_GROUP_COUNT_DISTINCT && aggType != GroupByMeta.TYPE_GROUP_COUNT_ANY)) {
            vMeta.setLength(subjMeta.getLength(), subjMeta.getPrecision());
        }
        data.agg[i] = v;
        data.aggMeta.addValueMeta(vMeta);
    }
    // 
    for (int i = 0; i < data.previousSums.length; i++) {
        data.previousSums[i] = null;
    }
    for (int i = 0; i < data.previousAvgCount.length; i++) {
        data.previousAvgCount[i] = 0L;
        data.previousAvgSum[i] = null;
    }
}

Example 18 with ValueMetaNumber

use of org.pentaho.di.core.row.value.ValueMetaNumber in project pentaho-kettle by pentaho.

the class GroupBy method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (GroupByMeta) smi;
    data = (GroupByData) sdi;
    // get row!
    Object[] r = getRow();
    if (first) {
        String val = getVariable(Const.KETTLE_AGGREGATION_ALL_NULLS_ARE_ZERO, "N");
        allNullsAreZero = ValueMetaBase.convertStringToBoolean(val);
        val = getVariable(Const.KETTLE_AGGREGATION_MIN_NULL_IS_VALUED, "N");
        minNullIsValued = ValueMetaBase.convertStringToBoolean(val);
        // What is the output looking like?
        // 
        data.inputRowMeta = getInputRowMeta();
        // 
        if (data.inputRowMeta == null) {
            data.inputRowMeta = getTransMeta().getPrevStepFields(getStepMeta());
        }
        data.outputRowMeta = data.inputRowMeta.clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Do all the work we can beforehand
        // Calculate indexes, loop up fields, etc.
        // 
        data.counts = new long[meta.getSubjectField().length];
        data.subjectnrs = new int[meta.getSubjectField().length];
        data.cumulativeSumSourceIndexes = new ArrayList<>();
        data.cumulativeSumTargetIndexes = new ArrayList<>();
        data.cumulativeAvgSourceIndexes = new ArrayList<>();
        data.cumulativeAvgTargetIndexes = new ArrayList<>();
        for (int i = 0; i < meta.getSubjectField().length; i++) {
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_COUNT_ANY) {
                data.subjectnrs[i] = 0;
            } else {
                data.subjectnrs[i] = data.inputRowMeta.indexOfValue(meta.getSubjectField()[i]);
            }
            if ((r != null) && (data.subjectnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.AggregateSubjectFieldCouldNotFound", meta.getSubjectField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM) {
                data.cumulativeSumSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeSumTargetIndexes.add(data.inputRowMeta.size() + i);
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE) {
                data.cumulativeAvgSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeAvgTargetIndexes.add(data.inputRowMeta.size() + i);
            }
        }
        data.previousSums = new Object[data.cumulativeSumTargetIndexes.size()];
        data.previousAvgSum = new Object[data.cumulativeAvgTargetIndexes.size()];
        data.previousAvgCount = new long[data.cumulativeAvgTargetIndexes.size()];
        data.groupnrs = new int[meta.getGroupField().length];
        for (int i = 0; i < meta.getGroupField().length; i++) {
            data.groupnrs[i] = data.inputRowMeta.indexOfValue(meta.getGroupField()[i]);
            if ((r != null) && (data.groupnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.GroupFieldCouldNotFound", meta.getGroupField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
        }
        // Create a metadata value for the counter Integers
        // 
        data.valueMetaInteger = new ValueMetaInteger("count");
        data.valueMetaNumber = new ValueMetaNumber("sum");
        // Initialize the group metadata
        // 
        initGroupMeta(data.inputRowMeta);
    }
    if (first || data.newBatch) {
        // Create a new group aggregate (init)
        // 
        newAggregate(r);
    }
    if (first) {
        // for speed: groupMeta+aggMeta
        // 
        data.groupAggMeta = new RowMeta();
        data.groupAggMeta.addRowMeta(data.groupMeta);
        data.groupAggMeta.addRowMeta(data.aggMeta);
    }
    if (r == null) {
        // no more input to be expected... (or none received in the first place)
        handleLastOfGroup();
        setOutputDone();
        return false;
    }
    if (first || data.newBatch) {
        first = false;
        data.newBatch = false;
        // copy the row to previous
        data.previous = data.inputRowMeta.cloneRow(r);
    } else {
        calcAggregate(data.previous);
        if (meta.passAllRows()) {
            addToBuffer(data.previous);
        }
    }
    if (!sameGroup(data.previous, r)) {
        if (meta.passAllRows()) {
            // Not the same group: close output (if any)
            closeOutput();
            // Get all rows from the buffer!
            data.groupResult = getAggregateResult();
            Object[] row = getRowFromBuffer();
            long lineNr = 0;
            while (row != null) {
                int size = data.inputRowMeta.size();
                row = RowDataUtil.addRowData(row, size, data.groupResult);
                size += data.groupResult.length;
                lineNr++;
                if (meta.isAddingLineNrInGroup() && !Utils.isEmpty(meta.getLineNrInGroupField())) {
                    Object lineNrValue = new Long(lineNr);
                    // ValueMetaInterface lineNrValueMeta = new ValueMeta(meta.getLineNrInGroupField(),
                    // ValueMetaInterface.TYPE_INTEGER);
                    // lineNrValueMeta.setLength(9);
                    row = RowDataUtil.addValueData(row, size, lineNrValue);
                    size++;
                }
                addCumulativeSums(row);
                addCumulativeAverages(row);
                putRow(data.outputRowMeta, row);
                row = getRowFromBuffer();
            }
            closeInput();
        } else {
            Object[] result = buildResult(data.previous);
            if (result != null) {
                // copy row to possible alternate rowset(s).
                putRow(data.groupAggMeta, result);
            }
        }
        // Create a new group aggregate (init)
        newAggregate(r);
    }
    data.previous = data.inputRowMeta.cloneRow(r);
    if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "GroupBy.LineNumber") + getLinesRead());
        }
    }
    return true;
}

Example 19 with ValueMetaNumber

use of org.pentaho.di.core.row.value.ValueMetaNumber in project pentaho-kettle by pentaho.

the class JsonInputTest method testLargeDoubles.

/**
 * PDI-10384 Huge numbers causing exception in JSON input step<br>
 */
@Test
public void testLargeDoubles() throws Exception {
    // legacy mode yields null for these
    for (String nbr : new String[] { "1e20", "2.05E20", "1.52999996e20" }) {
        final String ibgNbrInput = "{ \"number\": " + nbr + " }";
        testSimpleJsonPath("$.number", new ValueMetaNumber("not so big number"), new Object[][] { new Object[] { ibgNbrInput } }, new Object[][] { new Object[] { ibgNbrInput, Double.parseDouble(nbr) } });
    }
}

Example 20 with ValueMetaNumber

use of org.pentaho.di.core.row.value.ValueMetaNumber in project pentaho-kettle by pentaho.

the class JsonInputTest method testIndexList.

@Test
public void testIndexList() throws Exception {
    final String jsonInputField = getBasicTestJson();
    testSimpleJsonPath("$..book[1,3].price", new ValueMetaNumber("price"), new Object[][] { new Object[] { jsonInputField } }, new Object[][] { new Object[] { jsonInputField, 12.99 }, new Object[] { jsonInputField, 22.99 } });
}