Examples with RowMeta org.pentaho.di.core.row.RowMeta used on opensource projects

Example 66 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class TransMeta method getStepFields.

/**
 * Returns the fields that are emitted by a certain step.
 *
 * @param stepMeta
 *          The step to be queried.
 * @param targetStep
 *          the target step
 * @param monitor
 *          The progress monitor for progress dialog. (null if not used!)
 * @return A row containing the fields emitted.
 * @throws KettleStepException
 *           the kettle step exception
 */
public RowMetaInterface getStepFields(StepMeta stepMeta, StepMeta targetStep, ProgressMonitorListener monitor) throws KettleStepException {
    RowMetaInterface row = new RowMeta();
    if (stepMeta == null) {
        return row;
    }
    String fromToCacheEntry = stepMeta.getName() + (targetStep != null ? ("-" + targetStep.getName()) : "");
    RowMetaInterface rowMeta = stepsFieldsCache.get(fromToCacheEntry);
    if (rowMeta != null) {
        return rowMeta;
    }
    // 
    if (targetStep != null && stepMeta.isSendingErrorRowsToStep(targetStep)) {
        // The error rows are the same as the input rows for
        // the step but with the selected error fields added
        // 
        row = getPrevStepFields(stepMeta);
        // Add to this the error fields...
        StepErrorMeta stepErrorMeta = stepMeta.getStepErrorMeta();
        row.addRowMeta(stepErrorMeta.getErrorFields());
        // Store this row in the cache
        // 
        stepsFieldsCache.put(fromToCacheEntry, row);
        return row;
    }
    // Resume the regular program...
    List<StepMeta> prevSteps = getPreviousSteps(stepMeta);
    int nrPrevious = prevSteps.size();
    if (log.isDebug()) {
        log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.FromStepALookingAtPreviousStep", stepMeta.getName(), String.valueOf(nrPrevious)));
    }
    for (int i = 0; i < prevSteps.size(); i++) {
        StepMeta prevStepMeta = prevSteps.get(i);
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.CheckingStepTask.Title", prevStepMeta.getName()));
        }
        RowMetaInterface add = getStepFields(prevStepMeta, stepMeta, monitor);
        if (add == null) {
            add = new RowMeta();
        }
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.FoundFieldsToAdd") + add.toString());
        }
        if (i == 0) {
            row.addRowMeta(add);
        } else {
            // See if the add fields are not already in the row
            for (int x = 0; x < add.size(); x++) {
                ValueMetaInterface v = add.getValueMeta(x);
                ValueMetaInterface s = row.searchValueMeta(v.getName());
                if (s == null) {
                    row.addValueMeta(v);
                }
            }
        }
    }
    if (nrPrevious == 0 && stepMeta.getRemoteInputSteps().size() > 0) {
        // 
        for (RemoteStep remoteStep : stepMeta.getRemoteInputSteps()) {
            RowMetaInterface inputFields = remoteStep.getRowMeta();
            for (ValueMetaInterface inputField : inputFields.getValueMetaList()) {
                if (row.searchValueMeta(inputField.getName()) == null) {
                    row.addValueMeta(inputField);
                }
            }
        }
    }
    // Finally, see if we need to add/modify/delete fields with this step "name"
    rowMeta = getThisStepFields(stepMeta, targetStep, row, monitor);
    // Store this row in the cache
    // 
    stepsFieldsCache.put(fromToCacheEntry, rowMeta);
    return rowMeta;
}

Example 67 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class ExcelWriterStep method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (ExcelWriterStepMeta) smi;
    data = (ExcelWriterStepData) sdi;
    // get next row
    Object[] r = getRow();
    // first row initialization
    if (first) {
        first = false;
        if (r == null) {
            data.outputRowMeta = new RowMeta();
            data.inputRowMeta = new RowMeta();
        } else {
            data.outputRowMeta = getInputRowMeta().clone();
            data.inputRowMeta = getInputRowMeta().clone();
        }
        // if we are supposed to init the file up front, here we go
        if (!meta.isDoNotOpenNewFileInit()) {
            data.firstFileOpened = true;
            try {
                prepareNextOutputFile();
            } catch (KettleException e) {
                logError(BaseMessages.getString(PKG, "ExcelWriterStep.Exception.CouldNotPrepareFile", environmentSubstitute(meta.getFileName())));
                setErrors(1L);
                stopAll();
                return false;
            }
        }
        if (r != null) {
            // if we are supposed to init the file delayed, here we go
            if (meta.isDoNotOpenNewFileInit()) {
                data.firstFileOpened = true;
                prepareNextOutputFile();
            }
            // remember where the output fields are in the input row
            data.fieldnrs = new int[meta.getOutputFields().length];
            for (int i = 0; i < meta.getOutputFields().length; i++) {
                data.fieldnrs[i] = data.inputRowMeta.indexOfValue(meta.getOutputFields()[i].getName());
                if (data.fieldnrs[i] < 0) {
                    logError("Field [" + meta.getOutputFields()[i].getName() + "] couldn't be found in the input stream!");
                    setErrors(1);
                    stopAll();
                    return false;
                }
            }
            // remember where the comment fields are in the input row
            data.commentfieldnrs = new int[meta.getOutputFields().length];
            for (int i = 0; i < meta.getOutputFields().length; i++) {
                data.commentfieldnrs[i] = data.inputRowMeta.indexOfValue(meta.getOutputFields()[i].getCommentField());
                if (data.commentfieldnrs[i] < 0 && !Utils.isEmpty(meta.getOutputFields()[i].getCommentField())) {
                    logError("Comment Field [" + meta.getOutputFields()[i].getCommentField() + "] couldn't be found in the input stream!");
                    setErrors(1);
                    stopAll();
                    return false;
                }
            }
            // remember where the comment author fields are in the input row
            data.commentauthorfieldnrs = new int[meta.getOutputFields().length];
            for (int i = 0; i < meta.getOutputFields().length; i++) {
                data.commentauthorfieldnrs[i] = data.inputRowMeta.indexOfValue(meta.getOutputFields()[i].getCommentAuthorField());
                if (data.commentauthorfieldnrs[i] < 0 && !Utils.isEmpty(meta.getOutputFields()[i].getCommentAuthorField())) {
                    logError("Comment Author Field [" + meta.getOutputFields()[i].getCommentAuthorField() + "] couldn't be found in the input stream!");
                    setErrors(1);
                    stopAll();
                    return false;
                }
            }
            // remember where the link fields are in the input row
            data.linkfieldnrs = new int[meta.getOutputFields().length];
            for (int i = 0; i < meta.getOutputFields().length; i++) {
                data.linkfieldnrs[i] = data.inputRowMeta.indexOfValue(meta.getOutputFields()[i].getHyperlinkField());
                if (data.linkfieldnrs[i] < 0 && !Utils.isEmpty(meta.getOutputFields()[i].getHyperlinkField())) {
                    logError("Link Field [" + meta.getOutputFields()[i].getHyperlinkField() + "] couldn't be found in the input stream!");
                    setErrors(1);
                    stopAll();
                    return false;
                }
            }
        }
    }
    if (r != null) {
        // File Splitting Feature, is it time to create a new file?
        if (!meta.isAppendLines() && meta.getSplitEvery() > 0 && data.datalines > 0 && data.datalines % meta.getSplitEvery() == 0) {
            closeOutputFile();
            prepareNextOutputFile();
        }
        writeNextLine(r);
        incrementLinesOutput();
        data.datalines++;
        // pass on the row unchanged
        putRow(data.outputRowMeta, r);
        // Some basic logging
        if (checkFeedback(getLinesOutput())) {
            if (log.isBasic()) {
                logBasic("Linenr " + getLinesOutput());
            }
        }
        return true;
    } else {
        // after the last row, the (last) file is closed
        if (data.wb != null) {
            closeOutputFile();
        }
        setOutputDone();
        clearWorkbookMem();
        return false;
    }
}

Example 68 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class FilesFromResult method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    if (data.resultFilesList == null || getLinesRead() >= data.resultFilesList.size()) {
        setOutputDone();
        return false;
    }
    ResultFile resultFile = data.resultFilesList.get((int) getLinesRead());
    RowMetaAndData r = resultFile.getRow();
    if (first) {
        first = false;
        data.outputRowMeta = new RowMeta();
        smi.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
    }
    incrementLinesRead();
    // copy row to possible alternate
    putRow(data.outputRowMeta, r.getData());
    if (checkFeedback(getLinesRead())) {
        logBasic(BaseMessages.getString(PKG, "FilesFromResult.Log.LineNumber") + getLinesRead());
    }
    return true;
}

Example 69 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class FixedInput method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (FixedInputMeta) smi;
    data = (FixedInputData) sdi;
    if (first) {
        first = false;
        data.outputRowMeta = new RowMeta();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // The conversion logic for when the lazy conversion is turned of is simple:
        // Pretend it's a lazy conversion object anyway and get the native type during conversion.
        // 
        data.convertRowMeta = data.outputRowMeta.clone();
        for (ValueMetaInterface valueMeta : data.convertRowMeta.getValueMetaList()) {
            valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
        }
        if (meta.isHeaderPresent()) {
            // skip this row.
            readOneRow(false);
        }
    }
    Object[] outputRowData = readOneRow(true);
    if (outputRowData == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    // copy row to possible alternate rowset(s).
    putRow(data.outputRowMeta, outputRowData);
    if (checkFeedback(getLinesInput())) {
        logBasic(BaseMessages.getString(PKG, "FixedInput.Log.LineNumber", Long.toString(getLinesInput())));
    }
    return true;
}

Example 70 with RowMeta

use of org.pentaho.di.core.row.RowMeta in project pentaho-kettle by pentaho.

the class DimensionLookup method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (DimensionLookupMeta) smi;
    data = (DimensionLookupData) sdi;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        // signal end to receiver(s)
        setOutputDone();
        return false;
    }
    if (first) {
        first = false;
        data.schemaTable = meta.getDatabaseMeta().getQuotedSchemaTableCombination(data.realSchemaName, data.realTableName);
        data.inputRowMeta = getInputRowMeta().clone();
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Get the fields that need conversion to normal storage...
        // Modify the storage type of the input data...
        // 
        data.lazyList = new ArrayList<Integer>();
        for (int i = 0; i < data.inputRowMeta.size(); i++) {
            ValueMetaInterface valueMeta = data.inputRowMeta.getValueMeta(i);
            if (valueMeta.isStorageBinaryString()) {
                data.lazyList.add(i);
                valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            }
        }
        // The start date value column (if applicable)
        // 
        data.startDateFieldIndex = -1;
        if (data.startDateChoice == DimensionLookupMeta.START_DATE_ALTERNATIVE_COLUMN_VALUE) {
            data.startDateFieldIndex = data.inputRowMeta.indexOfValue(meta.getStartDateFieldName());
            if (data.startDateFieldIndex < 0) {
                throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.StartDateValueColumnNotFound", meta.getStartDateFieldName()));
            }
        }
        // Lookup values
        data.keynrs = new int[meta.getKeyStream().length];
        for (int i = 0; i < meta.getKeyStream().length; i++) {
            // logDetailed("Lookup values key["+i+"] --> "+key[i]+", row==null?"+(row==null));
            data.keynrs[i] = data.inputRowMeta.indexOfValue(meta.getKeyStream()[i]);
            if (data.keynrs[i] < 0) {
                // couldn't find field!
                throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.KeyFieldNotFound", meta.getKeyStream()[i]));
            }
        }
        // Return values
        data.fieldnrs = new int[meta.getFieldStream().length];
        for (int i = 0; meta.getFieldStream() != null && i < meta.getFieldStream().length; i++) {
            if (!DimensionLookupMeta.isUpdateTypeWithoutArgument(meta.isUpdate(), meta.getFieldUpdate()[i])) {
                data.fieldnrs[i] = data.outputRowMeta.indexOfValue(meta.getFieldStream()[i]);
                if (data.fieldnrs[i] < 0) {
                    throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.KeyFieldNotFound", meta.getFieldStream()[i]));
                }
            } else {
                data.fieldnrs[i] = -1;
            }
        }
        if (!meta.isUpdate() && meta.isPreloadingCache()) {
            preloadCache();
        } else {
            // 
            if (data.cacheKeyRowMeta == null) {
                // KEY : the natural key(s)
                // 
                data.cacheKeyRowMeta = new RowMeta();
                for (int i = 0; i < data.keynrs.length; i++) {
                    ValueMetaInterface key = data.inputRowMeta.getValueMeta(data.keynrs[i]);
                    data.cacheKeyRowMeta.addValueMeta(key.clone());
                }
                data.cache = new ByteArrayHashMap(meta.getCacheSize() > 0 ? meta.getCacheSize() : 5000, data.cacheKeyRowMeta);
            }
        }
        if (!Utils.isEmpty(meta.getDateField())) {
            data.datefieldnr = data.inputRowMeta.indexOfValue(meta.getDateField());
        } else {
            data.datefieldnr = -1;
        }
        // Initialize the start date value in case we don't have one in the input rows
        // 
        data.valueDateNow = determineDimensionUpdatedDate(r);
        determineTechKeyCreation();
        data.notFoundTk = new Long(meta.getDatabaseMeta().getNotFoundTK(isAutoIncrement()));
        if (getCopy() == 0) {
            checkDimZero();
        }
        setDimLookup(data.outputRowMeta);
    }
    // 
    for (int lazyFieldIndex : data.lazyList) {
        ValueMetaInterface valueMeta = getInputRowMeta().getValueMeta(lazyFieldIndex);
        r[lazyFieldIndex] = valueMeta.convertToNormalStorageType(r[lazyFieldIndex]);
    }
    try {
        // add new values to the row in rowset[0].
        Object[] outputRow = lookupValues(data.inputRowMeta, r);
        // copy row to output rowset(s);
        putRow(data.outputRowMeta, outputRow);
        if (checkFeedback(getLinesRead())) {
            if (log.isBasic()) {
                logBasic(BaseMessages.getString(PKG, "DimensionLookup.Log.LineNumber") + getLinesRead());
            }
        }
    } catch (KettleException e) {
        logError(BaseMessages.getString(PKG, "DimensionLookup.Log.StepCanNotContinueForErrors", e.getMessage()));
        logError(Const.getStackTracker(e));
        setErrors(1);
        stopAll();
        // signal end to receiver(s)
        setOutputDone();
        return false;
    }
    return true;
}