Examples with ValueMetaInteger org.pentaho.di.core.row.value.ValueMetaInteger used on opensource projects

Example 41 with ValueMetaInteger

use of org.pentaho.di.core.row.value.ValueMetaInteger in project pentaho-kettle by pentaho.

the class GroupBy method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (GroupByMeta) smi;
    data = (GroupByData) sdi;
    // get row!
    Object[] r = getRow();
    if (first) {
        String val = getVariable(Const.KETTLE_AGGREGATION_ALL_NULLS_ARE_ZERO, "N");
        allNullsAreZero = ValueMetaBase.convertStringToBoolean(val);
        val = getVariable(Const.KETTLE_AGGREGATION_MIN_NULL_IS_VALUED, "N");
        minNullIsValued = ValueMetaBase.convertStringToBoolean(val);
        // What is the output looking like?
        // 
        data.inputRowMeta = getInputRowMeta();
        // 
        if (data.inputRowMeta == null) {
            data.inputRowMeta = getTransMeta().getPrevStepFields(getStepMeta());
        }
        data.outputRowMeta = data.inputRowMeta.clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Do all the work we can beforehand
        // Calculate indexes, loop up fields, etc.
        // 
        data.counts = new long[meta.getSubjectField().length];
        data.subjectnrs = new int[meta.getSubjectField().length];
        data.cumulativeSumSourceIndexes = new ArrayList<>();
        data.cumulativeSumTargetIndexes = new ArrayList<>();
        data.cumulativeAvgSourceIndexes = new ArrayList<>();
        data.cumulativeAvgTargetIndexes = new ArrayList<>();
        for (int i = 0; i < meta.getSubjectField().length; i++) {
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_COUNT_ANY) {
                data.subjectnrs[i] = 0;
            } else {
                data.subjectnrs[i] = data.inputRowMeta.indexOfValue(meta.getSubjectField()[i]);
            }
            if ((r != null) && (data.subjectnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.AggregateSubjectFieldCouldNotFound", meta.getSubjectField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM) {
                data.cumulativeSumSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeSumTargetIndexes.add(data.inputRowMeta.size() + i);
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE) {
                data.cumulativeAvgSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeAvgTargetIndexes.add(data.inputRowMeta.size() + i);
            }
        }
        data.previousSums = new Object[data.cumulativeSumTargetIndexes.size()];
        data.previousAvgSum = new Object[data.cumulativeAvgTargetIndexes.size()];
        data.previousAvgCount = new long[data.cumulativeAvgTargetIndexes.size()];
        data.groupnrs = new int[meta.getGroupField().length];
        for (int i = 0; i < meta.getGroupField().length; i++) {
            data.groupnrs[i] = data.inputRowMeta.indexOfValue(meta.getGroupField()[i]);
            if ((r != null) && (data.groupnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.GroupFieldCouldNotFound", meta.getGroupField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
        }
        // Create a metadata value for the counter Integers
        // 
        data.valueMetaInteger = new ValueMetaInteger("count");
        data.valueMetaNumber = new ValueMetaNumber("sum");
        // Initialize the group metadata
        // 
        initGroupMeta(data.inputRowMeta);
    }
    if (first || data.newBatch) {
        // Create a new group aggregate (init)
        // 
        newAggregate(r);
    }
    if (first) {
        // for speed: groupMeta+aggMeta
        // 
        data.groupAggMeta = new RowMeta();
        data.groupAggMeta.addRowMeta(data.groupMeta);
        data.groupAggMeta.addRowMeta(data.aggMeta);
    }
    if (r == null) {
        // no more input to be expected... (or none received in the first place)
        handleLastOfGroup();
        setOutputDone();
        return false;
    }
    if (first || data.newBatch) {
        first = false;
        data.newBatch = false;
        // copy the row to previous
        data.previous = data.inputRowMeta.cloneRow(r);
    } else {
        calcAggregate(data.previous);
        if (meta.passAllRows()) {
            addToBuffer(data.previous);
        }
    }
    if (!sameGroup(data.previous, r)) {
        if (meta.passAllRows()) {
            // Not the same group: close output (if any)
            closeOutput();
            // Get all rows from the buffer!
            data.groupResult = getAggregateResult();
            Object[] row = getRowFromBuffer();
            long lineNr = 0;
            while (row != null) {
                int size = data.inputRowMeta.size();
                row = RowDataUtil.addRowData(row, size, data.groupResult);
                size += data.groupResult.length;
                lineNr++;
                if (meta.isAddingLineNrInGroup() && !Utils.isEmpty(meta.getLineNrInGroupField())) {
                    Object lineNrValue = new Long(lineNr);
                    // ValueMetaInterface lineNrValueMeta = new ValueMeta(meta.getLineNrInGroupField(),
                    // ValueMetaInterface.TYPE_INTEGER);
                    // lineNrValueMeta.setLength(9);
                    row = RowDataUtil.addValueData(row, size, lineNrValue);
                    size++;
                }
                addCumulativeSums(row);
                addCumulativeAverages(row);
                putRow(data.outputRowMeta, row);
                row = getRowFromBuffer();
            }
            closeInput();
        } else {
            Object[] result = buildResult(data.previous);
            if (result != null) {
                // copy row to possible alternate rowset(s).
                putRow(data.groupAggMeta, result);
            }
        }
        // Create a new group aggregate (init)
        newAggregate(r);
    }
    data.previous = data.inputRowMeta.cloneRow(r);
    if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "GroupBy.LineNumber") + getLinesRead());
        }
    }
    return true;
}

Example 42 with ValueMetaInteger

use of org.pentaho.di.core.row.value.ValueMetaInteger in project pentaho-kettle by pentaho.

the class GroupBy method newAggregate.

/**
 * used for junits in GroupByAggregationNullsTest
 *
 * @param r
 */
void newAggregate(Object[] r) {
    // Put all the counters at 0
    for (int i = 0; i < data.counts.length; i++) {
        data.counts[i] = 0;
    }
    data.distinctObjs = null;
    data.agg = new Object[data.subjectnrs.length];
    // sets all doubles to 0.0
    data.mean = new double[data.subjectnrs.length];
    data.aggMeta = new RowMeta();
    for (int i = 0; i < data.subjectnrs.length; i++) {
        ValueMetaInterface subjMeta = data.inputRowMeta.getValueMeta(data.subjectnrs[i]);
        Object v = null;
        ValueMetaInterface vMeta = null;
        int aggType = meta.getAggregateType()[i];
        switch(aggType) {
            case GroupByMeta.TYPE_GROUP_SUM:
            case GroupByMeta.TYPE_GROUP_AVERAGE:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE:
                if (subjMeta.isNumeric()) {
                    try {
                        vMeta = ValueMetaFactory.createValueMeta(meta.getAggregateField()[i], subjMeta.getType());
                    } catch (KettlePluginException e) {
                        vMeta = new ValueMetaNone(meta.getAggregateField()[i]);
                    }
                } else {
                    vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                }
                break;
            case GroupByMeta.TYPE_GROUP_MEDIAN:
            case GroupByMeta.TYPE_GROUP_PERCENTILE:
            case GroupByMeta.TYPE_GROUP_PERCENTILE_NEAREST_RANK:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                v = new ArrayList<Double>();
                break;
            case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION:
            case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION_SAMPLE:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_COUNT_DISTINCT:
            case GroupByMeta.TYPE_GROUP_COUNT_ANY:
            case GroupByMeta.TYPE_GROUP_COUNT_ALL:
                vMeta = new ValueMetaInteger(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_FIRST:
            case GroupByMeta.TYPE_GROUP_LAST:
            case GroupByMeta.TYPE_GROUP_FIRST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_LAST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_MIN:
            case GroupByMeta.TYPE_GROUP_MAX:
                vMeta = subjMeta.clone();
                vMeta.setName(meta.getAggregateField()[i]);
                v = r == null ? null : r[data.subjectnrs[i]];
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_COMMA:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_STRING:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            default:
                // TODO raise an error here because we cannot continue successfully maybe the UI should validate this
                break;
        }
        if ((subjMeta != null) && (aggType != GroupByMeta.TYPE_GROUP_COUNT_ALL && aggType != GroupByMeta.TYPE_GROUP_COUNT_DISTINCT && aggType != GroupByMeta.TYPE_GROUP_COUNT_ANY)) {
            vMeta.setLength(subjMeta.getLength(), subjMeta.getPrecision());
        }
        data.agg[i] = v;
        data.aggMeta.addValueMeta(vMeta);
    }
    // 
    for (int i = 0; i < data.previousSums.length; i++) {
        data.previousSums[i] = null;
    }
    for (int i = 0; i < data.previousAvgCount.length; i++) {
        data.previousAvgCount[i] = 0L;
        data.previousAvgSum[i] = null;
    }
}

Example 43 with ValueMetaInteger

use of org.pentaho.di.core.row.value.ValueMetaInteger in project pentaho-kettle by pentaho.

the class BaseMetadataInjectionTest method check.

/**
 * Check int property.
 */
protected void check(String propertyName, IntGetter getter) throws KettleException {
    ValueMetaInterface valueMetaString = new ValueMetaString("f");
    injector.setProperty(meta, propertyName, setValue(valueMetaString, "1"), "f");
    assertEquals(1, getter.get());
    injector.setProperty(meta, propertyName, setValue(valueMetaString, "45"), "f");
    assertEquals(45, getter.get());
    ValueMetaInterface valueMetaInteger = new ValueMetaInteger("f");
    injector.setProperty(meta, propertyName, setValue(valueMetaInteger, 1234L), "f");
    assertEquals(1234, getter.get());
    injector.setProperty(meta, propertyName, setValue(valueMetaInteger, (long) Integer.MAX_VALUE), "f");
    assertEquals(Integer.MAX_VALUE, getter.get());
    skipPropertyTest(propertyName);
}

Example 44 with ValueMetaInteger

use of org.pentaho.di.core.row.value.ValueMetaInteger in project pentaho-kettle by pentaho.

the class BaseMetadataInjectionTest method check.

/**
 * Check long property.
 */
protected void check(String propertyName, LongGetter getter) throws KettleException {
    ValueMetaInterface valueMetaString = new ValueMetaString("f");
    injector.setProperty(meta, propertyName, setValue(valueMetaString, "1"), "f");
    assertEquals(1, getter.get());
    injector.setProperty(meta, propertyName, setValue(valueMetaString, "45"), "f");
    assertEquals(45, getter.get());
    ValueMetaInterface valueMetaInteger = new ValueMetaInteger("f");
    injector.setProperty(meta, propertyName, setValue(valueMetaInteger, 1234L), "f");
    assertEquals(1234, getter.get());
    injector.setProperty(meta, propertyName, setValue(valueMetaInteger, Long.MAX_VALUE), "f");
    assertEquals(Long.MAX_VALUE, getter.get());
    skipPropertyTest(propertyName);
}

Example 45 with ValueMetaInteger

use of org.pentaho.di.core.row.value.ValueMetaInteger in project pentaho-kettle by pentaho.

the class GroupByMetaGetFieldsTest method setup.

@Before
public void setup() throws KettlePluginException {
    rowMeta = spy(new RowMeta());
    groupByMeta = spy(new GroupByMeta());
    mockStatic(ValueMetaFactory.class);
    when(ValueMetaFactory.createValueMeta(anyInt())).thenCallRealMethod();
    when(ValueMetaFactory.createValueMeta(anyString(), anyInt())).thenCallRealMethod();
    when(ValueMetaFactory.createValueMeta("maxDate", 3, -1, -1)).thenReturn(new ValueMetaDate("maxDate"));
    when(ValueMetaFactory.createValueMeta("minDate", 3, -1, -1)).thenReturn(new ValueMetaDate("minDate"));
    when(ValueMetaFactory.createValueMeta("countDate", 5, -1, -1)).thenReturn(new ValueMetaInteger("countDate"));
    when(ValueMetaFactory.getValueMetaName(3)).thenReturn("Date");
    when(ValueMetaFactory.getValueMetaName(5)).thenReturn("Integer");
}