Examples with KettleStepException org.pentaho.di.core.exception.KettleStepException used on opensource projects

Example 46 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class MultiMergeJoin method processFirstRow.

private boolean processFirstRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MultiMergeJoinMeta) smi;
    data = (MultiMergeJoinData) sdi;
    TransMeta transMeta = getTransMeta();
    TransHopMeta transHopMeta;
    StepIOMetaInterface stepIOMeta = meta.getStepIOMeta();
    List<StreamInterface> infoStreams = stepIOMeta.getInfoStreams();
    StreamInterface stream;
    StepMeta toStepMeta = meta.getParentStepMeta();
    StepMeta fromStepMeta;
    ArrayList<String> inputStepNameList = new ArrayList<String>();
    String[] inputStepNames = meta.getInputSteps();
    String inputStepName;
    for (int i = 0; i < infoStreams.size(); i++) {
        inputStepName = inputStepNames[i];
        stream = infoStreams.get(i);
        fromStepMeta = stream.getStepMeta();
        if (fromStepMeta == null) {
            // should not arrive here, shoud typically have been caught by init.
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Log.UnableToFindReferenceStream", inputStepName));
        }
        // check the hop
        transHopMeta = transMeta.findTransHop(fromStepMeta, toStepMeta, true);
        // there is no hop: this is unexpected.
        if (transHopMeta == null) {
            // should not arrive here, shoud typically have been caught by init.
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Log.UnableToFindReferenceStream", inputStepName));
        } else if (transHopMeta.isEnabled()) {
            inputStepNameList.add(inputStepName);
        } else {
            logDetailed(BaseMessages.getString(PKG, "MultiMergeJoin.Log.IgnoringStep", inputStepName));
        }
    }
    int streamSize = inputStepNameList.size();
    if (streamSize == 0) {
        return false;
    }
    String keyField;
    String[] keyFields;
    data.rowSets = new RowSet[streamSize];
    RowSet rowSet;
    Object[] row;
    data.rows = new Object[streamSize][];
    data.metas = new RowMetaInterface[streamSize];
    data.rowLengths = new int[streamSize];
    MultiMergeJoinData.QueueComparator comparator = new MultiMergeJoinData.QueueComparator(data);
    data.queue = new PriorityQueue<MultiMergeJoinData.QueueEntry>(streamSize, comparator);
    data.results = new ArrayList<List<Object[]>>(streamSize);
    MultiMergeJoinData.QueueEntry queueEntry;
    data.queueEntries = new MultiMergeJoinData.QueueEntry[streamSize];
    data.drainIndices = new int[streamSize];
    data.keyNrs = new int[streamSize][];
    data.dummy = new Object[streamSize][];
    RowMetaInterface rowMeta;
    data.outputRowMeta = new RowMeta();
    for (int i = 0, j = 0; i < inputStepNames.length; i++) {
        inputStepName = inputStepNames[i];
        if (!inputStepNameList.contains(inputStepName)) {
            // ignore step with disabled hop.
            continue;
        }
        queueEntry = new MultiMergeJoinData.QueueEntry();
        queueEntry.index = j;
        data.queueEntries[j] = queueEntry;
        data.results.add(new ArrayList<Object[]>());
        rowSet = findInputRowSet(inputStepName);
        if (rowSet == null) {
            throw new KettleException(BaseMessages.getString(PKG, "MultiMergeJoin.Exception.UnableToFindSpecifiedStep", inputStepName));
        }
        data.rowSets[j] = rowSet;
        row = getRowFrom(rowSet);
        data.rows[j] = row;
        if (row == null) {
            rowMeta = getTransMeta().getStepFields(inputStepName);
            data.metas[j] = rowMeta;
        } else {
            queueEntry.row = row;
            rowMeta = rowSet.getRowMeta();
            keyField = meta.getKeyFields()[i];
            String[] keyFieldParts = keyField.split(",");
            String keyFieldPart;
            data.keyNrs[j] = new int[keyFieldParts.length];
            for (int k = 0; k < keyFieldParts.length; k++) {
                keyFieldPart = keyFieldParts[k];
                data.keyNrs[j][k] = rowMeta.indexOfValue(keyFieldPart);
                if (data.keyNrs[j][k] < 0) {
                    String message = BaseMessages.getString(PKG, "MultiMergeJoin.Exception.UnableToFindFieldInReferenceStream", keyFieldPart, inputStepName);
                    logError(message);
                    throw new KettleStepException(message);
                }
            }
            data.metas[j] = rowMeta;
            data.queue.add(data.queueEntries[j]);
        }
        data.outputRowMeta.mergeRowMeta(rowMeta.clone());
        data.rowLengths[j] = rowMeta.size();
        data.dummy[j] = RowDataUtil.allocateRowData(rowMeta.size());
        j++;
    }
    return true;
}

Example 47 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class LDAPInputMeta method getFields.

@Override
public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    int i;
    for (i = 0; i < inputFields.length; i++) {
        LDAPInputField field = inputFields[i];
        int type = field.getType();
        if (type == ValueMetaInterface.TYPE_NONE) {
            type = ValueMetaInterface.TYPE_STRING;
        }
        try {
            ValueMetaInterface v = ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type);
            v.setLength(field.getLength(), field.getPrecision());
            v.setOrigin(name);
            r.addValueMeta(v);
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
    if (includeRowNumber) {
        ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(rowNumberField));
        v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
}

Example 48 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class LDIFInputMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    int i;
    for (i = 0; i < inputFields.length; i++) {
        LDIFInputField field = inputFields[i];
        int type = field.getType();
        if (type == ValueMetaInterface.TYPE_NONE) {
            type = ValueMetaInterface.TYPE_STRING;
        }
        try {
            ValueMetaInterface v = ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type);
            v.setLength(field.getLength(), field.getPrecision());
            v.setOrigin(name);
            r.addValueMeta(v);
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
    if (includeFilename) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(filenameField));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (includeRowNumber) {
        ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(rowNumberField));
        v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (includeContentType) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(contentTypeField));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (includeDN) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(DNField));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getShortFileNameField() != null && getShortFileNameField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getShortFileNameField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getExtensionField() != null && getExtensionField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getExtensionField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getPathField() != null && getPathField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getPathField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getSizeField() != null && getSizeField().length() > 0) {
        ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(getSizeField()));
        v.setOrigin(name);
        v.setLength(9);
        r.addValueMeta(v);
    }
    if (isHiddenField() != null && isHiddenField().length() > 0) {
        ValueMetaInterface v = new ValueMetaBoolean(space.environmentSubstitute(isHiddenField()));
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getLastModificationDateField() != null && getLastModificationDateField().length() > 0) {
        ValueMetaInterface v = new ValueMetaDate(space.environmentSubstitute(getLastModificationDateField()));
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getUriField() != null && getUriField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getUriField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getRootUriField() != null && getRootUriField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getRootUriField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
}

Example 49 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class LoadFileInputMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    if (!getIsInFields()) {
        r.clear();
    }
    int i;
    for (i = 0; i < inputFields.length; i++) {
        LoadFileInputField field = inputFields[i];
        int type = field.getType();
        switch(field.getElementType()) {
            case LoadFileInputField.ELEMENT_TYPE_FILECONTENT:
                if (type == ValueMetaInterface.TYPE_NONE) {
                    type = ValueMetaInterface.TYPE_STRING;
                }
                break;
            case LoadFileInputField.ELEMENT_TYPE_FILESIZE:
                if (type == ValueMetaInterface.TYPE_NONE) {
                    type = ValueMetaInterface.TYPE_INTEGER;
                }
                break;
            default:
                break;
        }
        try {
            ValueMetaInterface v = ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type);
            v.setLength(field.getLength());
            v.setPrecision(field.getPrecision());
            v.setConversionMask(field.getFormat());
            v.setCurrencySymbol(field.getCurrencySymbol());
            v.setDecimalSymbol(field.getDecimalSymbol());
            v.setGroupingSymbol(field.getGroupSymbol());
            v.setTrimType(field.getTrimType());
            v.setOrigin(name);
            r.addValueMeta(v);
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
    if (includeFilename) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(filenameField));
        v.setLength(250);
        v.setPrecision(-1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (includeRowNumber) {
        ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(rowNumberField));
        v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getShortFileNameField() != null && getShortFileNameField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getShortFileNameField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getExtensionField() != null && getExtensionField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getExtensionField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getPathField() != null && getPathField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getPathField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (isHiddenField() != null && isHiddenField().length() > 0) {
        ValueMetaInterface v = new ValueMetaBoolean(space.environmentSubstitute(isHiddenField()));
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getLastModificationDateField() != null && getLastModificationDateField().length() > 0) {
        ValueMetaInterface v = new ValueMetaDate(space.environmentSubstitute(getLastModificationDateField()));
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getUriField() != null && getUriField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getUriField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (getRootUriField() != null && getRootUriField().length() > 0) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getRootUriField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
}

Example 50 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class BaseStep method handleGetRow.

private Object[] handleGetRow() throws KettleException {
    // 
    while (paused.get() && !stopped.get()) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            throw new KettleStepException(e);
        }
    }
    if (stopped.get()) {
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "BaseStep.Log.StopLookingForMoreRows"));
        }
        stopAll();
        return null;
    }
    // Small startup check
    // 
    waitUntilTransformationIsStarted();
    // See if we need to open sockets to remote input steps...
    // 
    openRemoteInputStepSocketsOnce();
    RowSet inputRowSet = null;
    Object[] row = null;
    synchronized (inputRowSetsLock) {
        // 
        if (inputRowSets.isEmpty()) {
            return null;
        }
        // Do we need to switch to the next input stream?
        if (blockPointer >= NR_OF_ROWS_IN_BLOCK) {
            // 
            for (int r = 0; r < inputRowSets.size() && row == null; r++) {
                nextInputStream();
                inputRowSet = currentInputStream();
                row = inputRowSet.getRowImmediate();
            }
            if (row != null) {
                incrementLinesRead();
            }
        } else {
            // What's the current input stream?
            inputRowSet = currentInputStream();
        }
        // 
        if (isUsingThreadPriorityManagment() && !inputRowSet.isDone() && inputRowSet.size() <= lowerBufferBoundary && !isStopped()) {
            try {
                Thread.sleep(0, 1);
            } catch (InterruptedException e) {
            // Ignore sleep interruption exception
            }
        }
        // 
        while (row == null && !isStopped()) {
            // Get a row from the input in row set ...
            // Timeout immediately if nothing is there to read.
            // We will then switch to the next row set to read from...
            // 
            row = inputRowSet.getRowWait(1, TimeUnit.MILLISECONDS);
            if (row != null) {
                incrementLinesRead();
                blockPointer++;
            } else {
                // 
                if (inputRowSet.isDone()) {
                    row = inputRowSet.getRowWait(1, TimeUnit.MILLISECONDS);
                    if (row == null) {
                        inputRowSets.remove(currentInputRowSetNr);
                        if (inputRowSets.isEmpty()) {
                            // We're completely done.
                            return null;
                        }
                    } else {
                        incrementLinesRead();
                    }
                }
                nextInputStream();
                inputRowSet = currentInputStream();
            }
        }
        // 
        while (row == null && !stopped.get()) {
            // 
            if (inputRowSets.isEmpty()) {
                // We're done.
                return null;
            }
            nextInputStream();
            inputRowSet = currentInputStream();
            row = getRowFrom(inputRowSet);
        }
    }
    // or if prevSteps.length > 1 inputRowMeta can be changed
    if (inputRowMeta == null || prevSteps.length > 1) {
        inputRowMeta = inputRowSet.getRowMeta();
    }
    if (row != null) {
        // 
        if (trans.isSafeModeEnabled()) {
            transMeta.checkRowMixingStatically(stepMeta, null);
        }
        for (RowListener listener : rowListeners) {
            listener.rowReadEvent(inputRowMeta, row);
        }
    }
    // Check the rejection rates etc. as well.
    verifyRejectionRates();
    return row;
}