Examples with RowMeta org.pentaho.di.core.row.RowMeta used on opensource projects

Example 96 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class GetVariableMeta method getFields.

@Override
public void getFields(RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // Determine the maximum length...
    // 
    int length = -1;
    for (int i = 0; i < fieldDefinitions.length; i++) {
        String variableString = fieldDefinitions[i].getVariableString();
        if (variableString != null) {
            String string = space.environmentSubstitute(variableString);
            if (string.length() > length) {
                length = string.length();
            }
        }
    }
    RowMetaInterface row = new RowMeta();
    for (int i = 0; i < fieldDefinitions.length; i++) {
        ValueMetaInterface v;
        try {
            v = ValueMetaFactory.createValueMeta(fieldDefinitions[i].getFieldName(), fieldDefinitions[i].getFieldType());
        } catch (KettlePluginException e) {
            throw new KettleStepException(e);
        }
        int fieldLength = fieldDefinitions[i].getFieldLength();
        if (fieldLength < 0) {
            v.setLength(length);
        } else {
            v.setLength(fieldLength);
        }
        int fieldPrecision = fieldDefinitions[i].getFieldPrecision();
        if (fieldPrecision >= 0) {
            v.setPrecision(fieldPrecision);
        }
        v.setConversionMask(fieldDefinitions[i].getFieldFormat());
        v.setGroupingSymbol(fieldDefinitions[i].getGroup());
        v.setDecimalSymbol(fieldDefinitions[i].getDecimal());
        v.setCurrencySymbol(fieldDefinitions[i].getCurrency());
        v.setTrimType(fieldDefinitions[i].getTrimType());
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    inputRowMeta.mergeRowMeta(row, name);
}

Example 97 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class GroupBy method newAggregate.

/**
 * used for junits in GroupByAggregationNullsTest
 *
 * @param r
 */
void newAggregate(Object[] r) {
    // Put all the counters at 0
    for (int i = 0; i < data.counts.length; i++) {
        data.counts[i] = 0;
    }
    data.distinctObjs = null;
    data.agg = new Object[data.subjectnrs.length];
    // sets all doubles to 0.0
    data.mean = new double[data.subjectnrs.length];
    data.aggMeta = new RowMeta();
    for (int i = 0; i < data.subjectnrs.length; i++) {
        ValueMetaInterface subjMeta = data.inputRowMeta.getValueMeta(data.subjectnrs[i]);
        Object v = null;
        ValueMetaInterface vMeta = null;
        int aggType = meta.getAggregateType()[i];
        switch(aggType) {
            case GroupByMeta.TYPE_GROUP_SUM:
            case GroupByMeta.TYPE_GROUP_AVERAGE:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM:
            case GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE:
                if (subjMeta.isNumeric()) {
                    try {
                        vMeta = ValueMetaFactory.createValueMeta(meta.getAggregateField()[i], subjMeta.getType());
                    } catch (KettlePluginException e) {
                        vMeta = new ValueMetaNone(meta.getAggregateField()[i]);
                    }
                } else {
                    vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                }
                break;
            case GroupByMeta.TYPE_GROUP_MEDIAN:
            case GroupByMeta.TYPE_GROUP_PERCENTILE:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                v = new ArrayList<Double>();
                break;
            case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION:
                vMeta = new ValueMetaNumber(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_COUNT_DISTINCT:
            case GroupByMeta.TYPE_GROUP_COUNT_ANY:
            case GroupByMeta.TYPE_GROUP_COUNT_ALL:
                vMeta = new ValueMetaInteger(meta.getAggregateField()[i]);
                break;
            case GroupByMeta.TYPE_GROUP_FIRST:
            case GroupByMeta.TYPE_GROUP_LAST:
            case GroupByMeta.TYPE_GROUP_FIRST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_LAST_INCL_NULL:
            case GroupByMeta.TYPE_GROUP_MIN:
            case GroupByMeta.TYPE_GROUP_MAX:
                vMeta = subjMeta.clone();
                vMeta.setName(meta.getAggregateField()[i]);
                v = r == null ? null : r[data.subjectnrs[i]];
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_COMMA:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            case GroupByMeta.TYPE_GROUP_CONCAT_STRING:
                vMeta = new ValueMetaString(meta.getAggregateField()[i]);
                v = new StringBuilder();
                break;
            default:
                // TODO raise an error here because we cannot continue successfully maybe the UI should validate this
                break;
        }
        if ((subjMeta != null) && (aggType != GroupByMeta.TYPE_GROUP_COUNT_ALL && aggType != GroupByMeta.TYPE_GROUP_COUNT_DISTINCT && aggType != GroupByMeta.TYPE_GROUP_COUNT_ANY)) {
            vMeta.setLength(subjMeta.getLength(), subjMeta.getPrecision());
        }
        data.agg[i] = v;
        data.aggMeta.addValueMeta(vMeta);
    }
    // 
    for (int i = 0; i < data.previousSums.length; i++) {
        data.previousSums[i] = null;
    }
    for (int i = 0; i < data.previousAvgCount.length; i++) {
        data.previousAvgCount[i] = 0L;
        data.previousAvgSum[i] = null;
    }
}

Example 98 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class GroupBy method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (GroupByMeta) smi;
    data = (GroupByData) sdi;
    // get row!
    Object[] r = getRow();
    if (first) {
        String val = getVariable(Const.KETTLE_AGGREGATION_ALL_NULLS_ARE_ZERO, "N");
        allNullsAreZero = ValueMetaBase.convertStringToBoolean(val);
        val = getVariable(Const.KETTLE_AGGREGATION_MIN_NULL_IS_VALUED, "N");
        minNullIsValued = ValueMetaBase.convertStringToBoolean(val);
        // What is the output looking like?
        // 
        data.inputRowMeta = getInputRowMeta();
        // 
        if (data.inputRowMeta == null) {
            data.inputRowMeta = getTransMeta().getPrevStepFields(getStepMeta());
        }
        data.outputRowMeta = data.inputRowMeta.clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Do all the work we can beforehand
        // Calculate indexes, loop up fields, etc.
        // 
        data.counts = new long[meta.getSubjectField().length];
        data.subjectnrs = new int[meta.getSubjectField().length];
        data.cumulativeSumSourceIndexes = new ArrayList<>();
        data.cumulativeSumTargetIndexes = new ArrayList<>();
        data.cumulativeAvgSourceIndexes = new ArrayList<>();
        data.cumulativeAvgTargetIndexes = new ArrayList<>();
        for (int i = 0; i < meta.getSubjectField().length; i++) {
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_COUNT_ANY) {
                data.subjectnrs[i] = 0;
            } else {
                data.subjectnrs[i] = data.inputRowMeta.indexOfValue(meta.getSubjectField()[i]);
            }
            if ((r != null) && (data.subjectnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.AggregateSubjectFieldCouldNotFound", meta.getSubjectField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM) {
                data.cumulativeSumSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeSumTargetIndexes.add(data.inputRowMeta.size() + i);
            }
            if (meta.getAggregateType()[i] == GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE) {
                data.cumulativeAvgSourceIndexes.add(data.subjectnrs[i]);
                // The position of the target in the output row is the input row size + i
                // 
                data.cumulativeAvgTargetIndexes.add(data.inputRowMeta.size() + i);
            }
        }
        data.previousSums = new Object[data.cumulativeSumTargetIndexes.size()];
        data.previousAvgSum = new Object[data.cumulativeAvgTargetIndexes.size()];
        data.previousAvgCount = new long[data.cumulativeAvgTargetIndexes.size()];
        data.groupnrs = new int[meta.getGroupField().length];
        for (int i = 0; i < meta.getGroupField().length; i++) {
            data.groupnrs[i] = data.inputRowMeta.indexOfValue(meta.getGroupField()[i]);
            if ((r != null) && (data.groupnrs[i] < 0)) {
                logError(BaseMessages.getString(PKG, "GroupBy.Log.GroupFieldCouldNotFound", meta.getGroupField()[i]));
                setErrors(1);
                stopAll();
                return false;
            }
        }
        // Create a metadata value for the counter Integers
        // 
        data.valueMetaInteger = new ValueMetaInteger("count");
        data.valueMetaNumber = new ValueMetaNumber("sum");
        // Initialize the group metadata
        // 
        initGroupMeta(data.inputRowMeta);
    }
    if (first || data.newBatch) {
        // Create a new group aggregate (init)
        // 
        newAggregate(r);
    }
    if (first) {
        // for speed: groupMeta+aggMeta
        // 
        data.groupAggMeta = new RowMeta();
        data.groupAggMeta.addRowMeta(data.groupMeta);
        data.groupAggMeta.addRowMeta(data.aggMeta);
    }
    if (r == null) {
        // no more input to be expected... (or none received in the first place)
        handleLastOfGroup();
        setOutputDone();
        return false;
    }
    if (first || data.newBatch) {
        first = false;
        data.newBatch = false;
        // copy the row to previous
        data.previous = data.inputRowMeta.cloneRow(r);
    } else {
        calcAggregate(data.previous);
        if (meta.passAllRows()) {
            addToBuffer(data.previous);
        }
    }
    if (!sameGroup(data.previous, r)) {
        if (meta.passAllRows()) {
            // Not the same group: close output (if any)
            closeOutput();
            // Get all rows from the buffer!
            data.groupResult = getAggregateResult();
            Object[] row = getRowFromBuffer();
            long lineNr = 0;
            while (row != null) {
                int size = data.inputRowMeta.size();
                row = RowDataUtil.addRowData(row, size, data.groupResult);
                size += data.groupResult.length;
                lineNr++;
                if (meta.isAddingLineNrInGroup() && !Utils.isEmpty(meta.getLineNrInGroupField())) {
                    Object lineNrValue = new Long(lineNr);
                    // ValueMetaInterface lineNrValueMeta = new ValueMeta(meta.getLineNrInGroupField(),
                    // ValueMetaInterface.TYPE_INTEGER);
                    // lineNrValueMeta.setLength(9);
                    row = RowDataUtil.addValueData(row, size, lineNrValue);
                    size++;
                }
                addCumulativeSums(row);
                addCumulativeAverages(row);
                putRow(data.outputRowMeta, row);
                row = getRowFromBuffer();
            }
            closeInput();
        } else {
            Object[] result = buildResult(data.previous);
            if (result != null) {
                // copy row to possible alternate rowset(s).
                putRow(data.groupAggMeta, result);
            }
        }
        // Create a new group aggregate (init)
        newAggregate(r);
    }
    data.previous = data.inputRowMeta.cloneRow(r);
    if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "GroupBy.LineNumber") + getLinesRead());
        }
    }
    return true;
}

Example 99 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class GPBulkLoaderMeta method getSQLStatements.

@Override
public SQLStatement getSQLStatements(TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // default: nothing to do!
    SQLStatement retval = new SQLStatement(stepMeta.getName(), databaseMeta, null);
    if (databaseMeta != null) {
        if (prev != null && prev.size() > 0) {
            // Copy the row
            RowMetaInterface tableFields = new RowMeta();
            // Now change the field names
            for (int i = 0; i < fieldTable.length; i++) {
                ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
                if (v != null) {
                    ValueMetaInterface tableField = v.clone();
                    tableField.setName(fieldTable[i]);
                    tableFields.addValueMeta(tableField);
                } else {
                    throw new KettleStepException("Unable to find field [" + fieldStream[i] + "] in the input rows");
                }
            }
            if (!Utils.isEmpty(tableName)) {
                Database db = new Database(loggingObject, databaseMeta);
                db.shareVariablesWith(transMeta);
                try {
                    db.connect();
                    String schemaTable = databaseMeta.getQuotedSchemaTableCombination(transMeta.environmentSubstitute(schemaName), transMeta.environmentSubstitute(tableName));
                    String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);
                    if (sql.length() == 0) {
                        retval.setSQL(null);
                    } else {
                        retval.setSQL(sql);
                    }
                } catch (KettleException e) {
                    retval.setError(BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.ErrorOccurred") + e.getMessage());
                }
            } else {
                retval.setError(BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection"));
            }
        } else {
            retval.setError(BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.NotReceivingAnyFields"));
        }
    } else {
        retval.setError(BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.NoConnectionDefined"));
    }
    return retval;
}

Example 100 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class GPLoadMeta method getSQLStatements.

public SQLStatement getSQLStatements(TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // default: nothing to do!
    SQLStatement retval = new SQLStatement(stepMeta.getName(), databaseMeta, null);
    if (databaseMeta != null) {
        if (prev != null && prev.size() > 0) {
            // Copy the row
            RowMetaInterface tableFields = new RowMeta();
            // Now change the field names
            for (int i = 0; i < fieldTable.length; i++) {
                ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
                if (v != null) {
                    ValueMetaInterface tableField = v.clone();
                    tableField.setName(fieldTable[i]);
                    tableFields.addValueMeta(tableField);
                } else {
                    throw new KettleStepException("Unable to find field [" + fieldStream[i] + "] in the input rows");
                }
            }
            if (!Utils.isEmpty(tableName)) {
                Database db = new Database(loggingObject, databaseMeta);
                db.shareVariablesWith(transMeta);
                try {
                    db.connect();
                    String schemaTable = databaseMeta.getQuotedSchemaTableCombination(transMeta.environmentSubstitute(schemaName), transMeta.environmentSubstitute(tableName));
                    String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);
                    if (sql.length() == 0) {
                        retval.setSQL(null);
                    } else {
                        retval.setSQL(sql);
                    }
                } catch (KettleException e) {
                    retval.setError(BaseMessages.getString(PKG, "GPLoadMeta.GetSQL.ErrorOccurred") + e.getMessage());
                }
            } else {
                retval.setError(BaseMessages.getString(PKG, "GPLoadMeta.GetSQL.NoTableDefinedOnConnection"));
            }
        } else {
            retval.setError(BaseMessages.getString(PKG, "GPLoadMeta.GetSQL.NotReceivingAnyFields"));
        }
    } else {
        retval.setError(BaseMessages.getString(PKG, "GPLoadMeta.GetSQL.NoConnectionDefined"));
    }
    return retval;
}