Examples with RowMeta org.pentaho.di.core.row.RowMeta used on opensource projects

Example 86 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class CsvInput method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (CsvInputMeta) smi;
    data = (CsvInputData) sdi;
    if (first) {
        first = false;
        data.outputRowMeta = new RowMeta();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        if (data.filenames == null) {
            // We're expecting the list of filenames from the previous step(s)...
            // 
            getFilenamesFromPreviousSteps();
        }
        // We only run in parallel if we have at least one file to process
        // AND if we have more than one step copy running...
        // 
        data.parallel = meta.isRunningInParallel() && data.totalNumberOfSteps > 1;
        // The conversion logic for when the lazy conversion is turned of is simple:
        // Pretend it's a lazy conversion object anyway and get the native type during conversion.
        // 
        data.convertRowMeta = data.outputRowMeta.clone();
        for (ValueMetaInterface valueMeta : data.convertRowMeta.getValueMetaList()) {
            valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
        }
        // Calculate the indexes for the filename and row number fields
        // 
        data.filenameFieldIndex = -1;
        if (!Utils.isEmpty(meta.getFilenameField()) && meta.isIncludingFilename()) {
            data.filenameFieldIndex = meta.getInputFields().length;
        }
        data.rownumFieldIndex = -1;
        if (!Utils.isEmpty(meta.getRowNumField())) {
            data.rownumFieldIndex = meta.getInputFields().length;
            if (data.filenameFieldIndex >= 0) {
                data.rownumFieldIndex++;
            }
        }
        // 
        if (data.parallel) {
            prepareToRunInParallel();
        }
        // 
        if (!openNextFile()) {
            setOutputDone();
            // nothing to see here, move along...
            return false;
        }
    }
    // 
    if (data.parallel) {
        if (data.totalBytesRead >= data.blockToRead) {
            // stop reading
            setOutputDone();
            return false;
        }
    }
    try {
        // get row, set busy!
        Object[] outputRowData = readOneRow(false, false);
        // no more input to be expected...
        if (outputRowData == null) {
            if (openNextFile()) {
                // try again on the next loop...
                return true;
            } else {
                // last file, end here
                setOutputDone();
                return false;
            }
        } else {
            // copy row to possible alternate rowset(s).
            putRow(data.outputRowMeta, outputRowData);
            if (checkFeedback(getLinesInput())) {
                if (log.isBasic()) {
                    logBasic(BaseMessages.getString(PKG, "CsvInput.Log.LineNumber", Long.toString(getLinesInput())));
                }
            }
        }
    } catch (KettleConversionException e) {
        if (getStepMeta().isDoingErrorHandling()) {
            StringBuilder errorDescriptions = new StringBuilder(100);
            StringBuilder errorFields = new StringBuilder(50);
            for (int i = 0; i < e.getCauses().size(); i++) {
                if (i > 0) {
                    errorDescriptions.append(", ");
                    errorFields.append(", ");
                }
                errorDescriptions.append(e.getCauses().get(i).getMessage());
                errorFields.append(e.getFields().get(i).toStringMeta());
            }
            putError(data.outputRowMeta, e.getRowData(), e.getCauses().size(), errorDescriptions.toString(), errorFields.toString(), "CSVINPUT001");
        } else {
            // 
            throw new KettleException(e.getMessage(), e.getCauses().get(0));
        }
    }
    return true;
}

Example 87 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class CubeInputMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    GZIPInputStream fis = null;
    DataInputStream dis = null;
    try {
        InputStream is = KettleVFS.getInputStream(space.environmentSubstitute(filename), space);
        fis = new GZIPInputStream(is);
        dis = new DataInputStream(fis);
        RowMetaInterface add = new RowMeta(dis);
        for (int i = 0; i < add.size(); i++) {
            add.getValueMeta(i).setOrigin(name);
        }
        r.mergeRowMeta(add);
    } catch (KettleFileException kfe) {
        throw new KettleStepException(BaseMessages.getString(PKG, "CubeInputMeta.Exception.UnableToReadMetaData"), kfe);
    } catch (IOException e) {
        throw new KettleStepException(BaseMessages.getString(PKG, "CubeInputMeta.Exception.ErrorOpeningOrReadingCubeFile"), e);
    } finally {
        try {
            if (fis != null) {
                fis.close();
            }
            if (dis != null) {
                dis.close();
            }
        } catch (IOException ioe) {
            throw new KettleStepException(BaseMessages.getString(PKG, "CubeInputMeta.Exception.UnableToCloseCubeFile"), ioe);
        }
    }
}

Example 88 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class DatabaseJoin method lookupValues.

private synchronized void lookupValues(RowMetaInterface rowMeta, Object[] rowData) throws KettleException {
    if (first) {
        first = false;
        data.outputRowMeta = rowMeta.clone();
        meta.getFields(data.outputRowMeta, getStepname(), new RowMetaInterface[] { meta.getTableFields() }, null, this, repository, metaStore);
        data.lookupRowMeta = new RowMeta();
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "DatabaseJoin.Log.CheckingRow") + rowMeta.getString(rowData));
        }
        data.keynrs = new int[meta.getParameterField().length];
        for (int i = 0; i < meta.getParameterField().length; i++) {
            data.keynrs[i] = rowMeta.indexOfValue(meta.getParameterField()[i]);
            if (data.keynrs[i] < 0) {
                throw new KettleStepException(BaseMessages.getString(PKG, "DatabaseJoin.Exception.FieldNotFound", meta.getParameterField()[i]));
            }
            data.lookupRowMeta.addValueMeta(rowMeta.getValueMeta(data.keynrs[i]).clone());
        }
    }
    // Construct the parameters row...
    Object[] lookupRowData = new Object[data.lookupRowMeta.size()];
    for (int i = 0; i < data.keynrs.length; i++) {
        lookupRowData[i] = rowData[data.keynrs[i]];
    }
    // Set the values on the prepared statement (for faster exec.)
    ResultSet rs = data.db.openQuery(data.pstmt, data.lookupRowMeta, lookupRowData);
    // Get a row from the database...
    // 
    Object[] add = data.db.getRow(rs);
    RowMetaInterface addMeta = data.db.getReturnRowMeta();
    incrementLinesInput();
    int counter = 0;
    while (add != null && (meta.getRowLimit() == 0 || counter < meta.getRowLimit())) {
        counter++;
        Object[] newRow = RowDataUtil.resizeArray(rowData, data.outputRowMeta.size());
        int newIndex = rowMeta.size();
        for (int i = 0; i < addMeta.size(); i++) {
            newRow[newIndex++] = add[i];
        }
        // we have to clone, otherwise we only get the last new value
        putRow(data.outputRowMeta, data.outputRowMeta.cloneRow(newRow));
        if (log.isRowLevel()) {
            logRowlevel(BaseMessages.getString(PKG, "DatabaseJoin.Log.PutoutRow") + data.outputRowMeta.getString(newRow));
        }
        // Get a new row
        if (meta.getRowLimit() == 0 || counter < meta.getRowLimit()) {
            add = data.db.getRow(rs);
            incrementLinesInput();
        }
    }
    // Nothing found? Perhaps we have to put something out after all?
    if (counter == 0 && meta.isOuterJoin()) {
        if (data.notfound == null) {
            // Just return null values for all values...
            // 
            data.notfound = new Object[data.db.getReturnRowMeta().size()];
        }
        Object[] newRow = RowDataUtil.resizeArray(rowData, data.outputRowMeta.size());
        int newIndex = rowMeta.size();
        for (int i = 0; i < data.notfound.length; i++) {
            newRow[newIndex++] = data.notfound[i];
        }
        putRow(data.outputRowMeta, newRow);
    }
    data.db.closeQuery(rs);
}

Example 89 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class DatabaseJoinMeta method getTableFields.

@Override
public RowMetaInterface getTableFields() {
    // Build a dummy parameter row...
    // 
    RowMetaInterface param = new RowMeta();
    for (int i = 0; i < parameterField.length; i++) {
        ValueMetaInterface v;
        try {
            v = ValueMetaFactory.createValueMeta(parameterField[i], parameterType[i]);
        } catch (KettlePluginException e) {
            v = new ValueMetaNone(parameterField[i]);
        }
        param.addValueMeta(v);
    }
    RowMetaInterface fields = null;
    if (databaseMeta != null) {
        Database db = new Database(loggingObject, databaseMeta);
        // Keep track of this one for cancelQuery
        databases = new Database[] { db };
        try {
            db.connect();
            fields = db.getQueryFields(databaseMeta.environmentSubstitute(sql), true, param, new Object[param.size()]);
        } catch (KettleDatabaseException dbe) {
            logError(BaseMessages.getString(PKG, "DatabaseJoinMeta.Log.DatabaseErrorOccurred") + dbe.getMessage());
        } finally {
            db.disconnect();
        }
    }
    return fields;
}

Example 90 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class DatabaseLookup method initLookupMeta.

private void initLookupMeta() throws KettleException {
    // Count the number of values in the lookup as well as the metadata to send along with it.
    // 
    data.lookupMeta = new RowMeta();
    for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
        if (data.keynrs[i] >= 0) {
            ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs[i]);
            // Try to convert type if needed in a clone, we don't want to
            // change the type in the original row
            // 
            ValueMetaInterface value = ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
            data.lookupMeta.addValueMeta(value);
        }
        if (data.keynrs2[i] >= 0) {
            ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs2[i]);
            // Try to convert type if needed in a clone, we don't want to
            // change the type in the original row
            // 
            ValueMetaInterface value = ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
            data.lookupMeta.addValueMeta(value);
        }
    }
}