Examples with ByteArrayHashMap org.pentaho.di.core.hash.ByteArrayHashMap used on opensource projects

Example 1 with ByteArrayHashMap

use of org.pentaho.di.core.hash.ByteArrayHashMap in project pentaho-kettle by pentaho.

the class DimensionLookup method processRow.

@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (DimensionLookupMeta) smi;
    data = (DimensionLookupData) sdi;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        // signal end to receiver(s)
        setOutputDone();
        return false;
    }
    if (first) {
        first = false;
        data.schemaTable = meta.getDatabaseMeta().getQuotedSchemaTableCombination(data.realSchemaName, data.realTableName);
        data.inputRowMeta = getInputRowMeta().clone();
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Get the fields that need conversion to normal storage...
        // Modify the storage type of the input data...
        // 
        data.lazyList = new ArrayList<Integer>();
        for (int i = 0; i < data.inputRowMeta.size(); i++) {
            ValueMetaInterface valueMeta = data.inputRowMeta.getValueMeta(i);
            if (valueMeta.isStorageBinaryString()) {
                data.lazyList.add(i);
                valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            }
        }
        // The start date value column (if applicable)
        // 
        data.startDateFieldIndex = -1;
        if (data.startDateChoice == DimensionLookupMeta.START_DATE_ALTERNATIVE_COLUMN_VALUE) {
            data.startDateFieldIndex = data.inputRowMeta.indexOfValue(meta.getStartDateFieldName());
            if (data.startDateFieldIndex < 0) {
                throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.StartDateValueColumnNotFound", meta.getStartDateFieldName()));
            }
        }
        // Lookup values
        data.keynrs = new int[meta.getKeyStream().length];
        for (int i = 0; i < meta.getKeyStream().length; i++) {
            // logDetailed("Lookup values key["+i+"] --> "+key[i]+", row==null?"+(row==null));
            data.keynrs[i] = data.inputRowMeta.indexOfValue(meta.getKeyStream()[i]);
            if (data.keynrs[i] < 0) {
                // couldn't find field!
                throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.KeyFieldNotFound", meta.getKeyStream()[i]));
            }
        }
        // Return values
        data.fieldnrs = new int[meta.getFieldStream().length];
        for (int i = 0; meta.getFieldStream() != null && i < meta.getFieldStream().length; i++) {
            if (!DimensionLookupMeta.isUpdateTypeWithoutArgument(meta.isUpdate(), meta.getFieldUpdate()[i])) {
                data.fieldnrs[i] = data.outputRowMeta.indexOfValue(meta.getFieldStream()[i]);
                if (data.fieldnrs[i] < 0) {
                    throw new KettleStepException(BaseMessages.getString(PKG, "DimensionLookup.Exception.KeyFieldNotFound", meta.getFieldStream()[i]));
                }
            } else {
                data.fieldnrs[i] = -1;
            }
        }
        if (!meta.isUpdate() && meta.isPreloadingCache()) {
            preloadCache();
        } else {
            // 
            if (data.cacheKeyRowMeta == null) {
                // KEY : the natural key(s)
                // 
                data.cacheKeyRowMeta = new RowMeta();
                for (int i = 0; i < data.keynrs.length; i++) {
                    ValueMetaInterface key = data.inputRowMeta.getValueMeta(data.keynrs[i]);
                    data.cacheKeyRowMeta.addValueMeta(key.clone());
                }
                data.cache = new ByteArrayHashMap(meta.getCacheSize() > 0 ? meta.getCacheSize() : 5000, data.cacheKeyRowMeta);
            }
        }
        if (!Utils.isEmpty(meta.getDateField())) {
            data.datefieldnr = data.inputRowMeta.indexOfValue(meta.getDateField());
        } else {
            data.datefieldnr = -1;
        }
        // Initialize the start date value in case we don't have one in the input rows
        // 
        data.valueDateNow = determineDimensionUpdatedDate(r);
        determineTechKeyCreation();
        data.notFoundTk = new Long(meta.getDatabaseMeta().getNotFoundTK(isAutoIncrement()));
        if (getCopy() == 0) {
            checkDimZero();
        }
        setDimLookup(data.outputRowMeta);
    }
    // 
    for (int lazyFieldIndex : data.lazyList) {
        ValueMetaInterface valueMeta = getInputRowMeta().getValueMeta(lazyFieldIndex);
        r[lazyFieldIndex] = valueMeta.convertToNormalStorageType(r[lazyFieldIndex]);
    }
    try {
        // add new values to the row in rowset[0].
        Object[] outputRow = lookupValues(data.inputRowMeta, r);
        // copy row to output rowset(s);
        putRow(data.outputRowMeta, outputRow);
        if (checkFeedback(getLinesRead())) {
            if (log.isBasic()) {
                logBasic(BaseMessages.getString(PKG, "DimensionLookup.Log.LineNumber") + getLinesRead());
            }
        }
    } catch (KettleException e) {
        logError(BaseMessages.getString(PKG, "DimensionLookup.Log.StepCanNotContinueForErrors", e.getMessage()));
        logError(Const.getStackTracker(e));
        setErrors(1);
        stopAll();
        // signal end to receiver(s)
        setOutputDone();
        return false;
    }
    return true;
}