Examples with KettleValueException org.pentaho.di.core.exception.KettleValueException used on opensource projects

Example 16 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class GetXMLDataStepAnalyzer method getInputRowMetaInterfaces.

@Override
protected Map<String, RowMetaInterface> getInputRowMetaInterfaces(GetXMLDataMeta meta) {
    Map<String, RowMetaInterface> inputRows = getInputFields(meta);
    if (inputRows == null) {
        inputRows = new HashMap<>();
    }
    // Get some boolean flags from the meta for easier access
    boolean isInFields = meta.isInFields();
    boolean isAFile = meta.getIsAFile();
    boolean isAUrl = meta.isReadUrl();
    // only add resource fields if we are NOT getting the xml or file from a field
    if (!isInFields || isAFile || isAUrl) {
        RowMetaInterface stepFields = getOutputFields(meta);
        RowMetaInterface clone = stepFields.clone();
        // if there are previous steps providing data, we should remove them from the set of "resource" fields
        for (RowMetaInterface rowMetaInterface : inputRows.values()) {
            for (ValueMetaInterface valueMetaInterface : rowMetaInterface.getValueMetaList()) {
                try {
                    clone.removeValueMeta(valueMetaInterface.getName());
                } catch (KettleValueException e) {
                // could not find it in the output, skip it
                }
            }
        }
        inputRows.put(RESOURCE, clone);
    }
    return inputRows;
}

Example 17 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class BaseStep method specialPartitioning.

private void specialPartitioning(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
    if (nextStepPartitioningMeta == null) {
        // Look up the partitioning of the next step.
        // This is the case for non-clustered partitioning...
        // 
        List<StepMeta> nextSteps = transMeta.findNextSteps(stepMeta);
        if (nextSteps.size() > 0) {
            nextStepPartitioningMeta = nextSteps.get(0).getStepPartitioningMeta();
        }
    // TODO: throw exception if we're not partitioning yet.
    // For now it throws a NP Exception.
    }
    int partitionNr;
    try {
        partitionNr = nextStepPartitioningMeta.getPartition(rowMeta, row);
    } catch (KettleException e) {
        throw new KettleStepException("Unable to convert a value to integer while calculating the partition number", e);
    }
    RowSet selectedRowSet = null;
    if (clusteredPartitioningFirst) {
        clusteredPartitioningFirst = false;
        // We are only running remotely if both the distribution is there AND if the distribution is actually contains
        // something.
        // 
        clusteredPartitioning = transMeta.getSlaveStepCopyPartitionDistribution() != null && !transMeta.getSlaveStepCopyPartitionDistribution().getDistribution().isEmpty();
    }
    // 
    if (clusteredPartitioning) {
        // 
        if (partitionNrRowSetList == null) {
            partitionNrRowSetList = new RowSet[outputRowSets.size()];
            // The distribution is calculated during transformation split
            // The slave-step-copy distribution is passed onto the slave transformation
            // 
            SlaveStepCopyPartitionDistribution distribution = transMeta.getSlaveStepCopyPartitionDistribution();
            String nextPartitionSchemaName = TransSplitter.createPartitionSchemaNameFromTarget(nextStepPartitioningMeta.getPartitionSchema().getName());
            for (RowSet outputRowSet : outputRowSets) {
                try {
                    // Look at the pre-determined distribution, decided at "transformation split" time.
                    // 
                    int partNr = distribution.getPartition(outputRowSet.getRemoteSlaveServerName(), nextPartitionSchemaName, outputRowSet.getDestinationStepCopy());
                    if (partNr < 0) {
                        throw new KettleStepException("Unable to find partition using rowset data, slave=" + outputRowSet.getRemoteSlaveServerName() + ", partition schema=" + nextStepPartitioningMeta.getPartitionSchema().getName() + ", copy=" + outputRowSet.getDestinationStepCopy());
                    }
                    partitionNrRowSetList[partNr] = outputRowSet;
                } catch (NullPointerException e) {
                    throw (e);
                }
            }
        }
        // 
        if (partitionNr < partitionNrRowSetList.length) {
            selectedRowSet = partitionNrRowSetList[partitionNr];
        } else {
            String rowsets = "";
            for (RowSet rowSet : partitionNrRowSetList) {
                rowsets += "[" + rowSet.toString() + "] ";
            }
            throw new KettleStepException("Internal error: the referenced partition nr '" + partitionNr + "' is higher than the maximum of '" + (partitionNrRowSetList.length - 1) + ".  The available row sets are: {" + rowsets + "}");
        }
        if (selectedRowSet == null) {
            logBasic(BaseMessages.getString(PKG, "BaseStep.TargetRowsetIsNotAvailable", partitionNr));
        } else {
            // Wait
            putRowToRowSet(selectedRowSet, rowMeta, row);
            incrementLinesWritten();
            if (log.isRowLevel()) {
                try {
                    logRowlevel("Partitioned #" + partitionNr + " to " + selectedRowSet + ", row=" + rowMeta.getString(row));
                } catch (KettleValueException e) {
                    throw new KettleStepException(e);
                }
            }
        }
    } else {
        // Local partitioning...
        // Put the row forward to the next step according to the partition rule.
        // 
        // Count of partitioned row at one step
        int partCount = ((BasePartitioner) nextStepPartitioningMeta.getPartitioner()).getNrPartitions();
        for (int i = 0; i < nextSteps.length; i++) {
            selectedRowSet = outputRowSets.get(partitionNr + i * partCount);
            if (selectedRowSet == null) {
                logBasic(BaseMessages.getString(PKG, "BaseStep.TargetRowsetIsNotAvailable", partitionNr));
            } else {
                // Wait
                putRowToRowSet(selectedRowSet, rowMeta, row);
                incrementLinesWritten();
                if (log.isRowLevel()) {
                    try {
                        logRowlevel(BaseMessages.getString(PKG, "BaseStep.PartitionedToRow", partitionNr, selectedRowSet, rowMeta.getString(row)));
                    } catch (KettleValueException e) {
                        throw new KettleStepException(e);
                    }
                }
            }
        }
    }
}

Example 18 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class BaseStep method noPartitioning.

private void noPartitioning(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
    if (distributed) {
        if (rowDistribution != null) {
            // Plugin defined row distribution!
            // 
            rowDistribution.distributeRow(rowMeta, row, this);
            incrementLinesWritten();
        } else {
            // ROUND ROBIN DISTRIBUTION:
            // --------------------------
            // Copy the row to the "next" output rowset.
            // We keep the next one in out_handling
            // 
            RowSet rs = outputRowSets.get(currentOutputRowSetNr);
            // 
            if (isUsingThreadPriorityManagment() && !rs.isDone() && rs.size() >= upperBufferBoundary && !isStopped()) {
                try {
                    Thread.sleep(0, 1);
                } catch (InterruptedException e) {
                // Ignore sleep interruption exception
                }
            }
            // Loop until we find room in the target rowset
            // 
            putRowToRowSet(rs, rowMeta, row);
            incrementLinesWritten();
            // 
            if (outputRowSets.size() > 1) {
                currentOutputRowSetNr++;
                if (currentOutputRowSetNr >= outputRowSets.size()) {
                    currentOutputRowSetNr = 0;
                }
            }
        }
    } else {
        // Copy to the row in the other output rowsets...
        for (int i = 1; i < outputRowSets.size(); i++) {
            // start at 1
            RowSet rs = outputRowSets.get(i);
            // 
            if (isUsingThreadPriorityManagment() && !rs.isDone() && rs.size() >= upperBufferBoundary && !isStopped()) {
                try {
                    Thread.sleep(0, 1);
                } catch (InterruptedException e) {
                // Ignore sleep interruption exception
                }
            }
            try {
                // Loop until we find room in the target rowset
                // 
                putRowToRowSet(rs, rowMeta, rowMeta.cloneRow(row));
                incrementLinesWritten();
            } catch (KettleValueException e) {
                throw new KettleStepException("Unable to clone row while copying rows to multiple target steps", e);
            }
        }
        // set row in first output rowset
        // 
        RowSet rs = outputRowSets.get(0);
        putRowToRowSet(rs, rowMeta, row);
        incrementLinesWritten();
    }
}

Example 19 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class FieldSplitter method splitField.

private Object[] splitField(Object[] r) throws KettleException {
    if (first) {
        first = false;
        // get the RowMeta
        data.previousMeta = getInputRowMeta().clone();
        // search field
        data.fieldnr = data.previousMeta.indexOfValue(meta.getSplitField());
        if (data.fieldnr < 0) {
            throw new KettleValueException(BaseMessages.getString(PKG, "FieldSplitter.Log.CouldNotFindFieldToSplit", meta.getSplitField()));
        }
        // only String type allowed
        if (!data.previousMeta.getValueMeta(data.fieldnr).isString()) {
            throw new KettleValueException((BaseMessages.getString(PKG, "FieldSplitter.Log.SplitFieldNotValid", meta.getSplitField())));
        }
        // prepare the outputMeta
        // 
        data.outputMeta = getInputRowMeta().clone();
        meta.getFields(data.outputMeta, getStepname(), null, null, this, repository, metaStore);
        // Now create objects to do string to data type conversion...
        // 
        data.conversionMeta = data.outputMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
        data.delimiter = environmentSubstitute(meta.getDelimiter());
        data.enclosure = environmentSubstitute(meta.getEnclosure());
    }
    // reserve room
    Object[] outputRow = RowDataUtil.allocateRowData(data.outputMeta.size());
    int nrExtraFields = meta.getFieldID().length - 1;
    System.arraycopy(r, 0, outputRow, 0, data.fieldnr);
    System.arraycopy(r, data.fieldnr + 1, outputRow, data.fieldnr + 1 + nrExtraFields, data.previousMeta.size() - (data.fieldnr + 1));
    // OK, now we have room in the middle to place the fields...
    // 
    // Named values info.id[0] not filled in!
    final boolean selectFieldById = (meta.getFieldID().length > 0) && (meta.getFieldID()[0] != null) && (meta.getFieldID()[0].length() > 0);
    if (log.isDebug()) {
        if (selectFieldById) {
            logDebug(BaseMessages.getString(PKG, "FieldSplitter.Log.UsingIds"));
        } else {
            logDebug(BaseMessages.getString(PKG, "FieldSplitter.Log.UsingPositionOfValue"));
        }
    }
    String valueToSplit = data.previousMeta.getString(r, data.fieldnr);
    boolean removeEnclosure = Boolean.valueOf(System.getProperties().getProperty(Const.KETTLE_SPLIT_FIELDS_REMOVE_ENCLOSURE, "false"));
    String[] valueParts = Const.splitString(valueToSplit, data.delimiter, data.enclosure, removeEnclosure);
    int prev = 0;
    for (int i = 0; i < meta.getFieldsCount(); i++) {
        String rawValue = null;
        if (selectFieldById) {
            for (String part : valueParts) {
                if (part.startsWith(meta.getFieldID()[i])) {
                    // Optionally remove the id
                    if (meta.getFieldRemoveID()[i]) {
                        rawValue = part.substring(meta.getFieldID()[i].length());
                    } else {
                        rawValue = part;
                    }
                    break;
                }
            }
            if (log.isDebug()) {
                logDebug(BaseMessages.getString(PKG, "FieldSplitter.Log.SplitInfo") + rawValue);
            }
        } else {
            rawValue = (valueParts == null || i >= valueParts.length) ? null : valueParts[i];
            prev += (rawValue == null ? 0 : rawValue.length()) + data.delimiter.length();
            if (log.isDebug()) {
                logDebug(BaseMessages.getString(PKG, "FieldSplitter.Log.SplitFieldsInfo", rawValue, String.valueOf(prev)));
            }
        }
        Object value;
        try {
            ValueMetaInterface valueMeta = data.outputMeta.getValueMeta(data.fieldnr + i);
            ValueMetaInterface conversionValueMeta = data.conversionMeta.getValueMeta(data.fieldnr + i);
            if (rawValue != null && valueMeta.isNull(rawValue)) {
                rawValue = null;
            }
            value = valueMeta.convertDataFromString(rawValue, conversionValueMeta, meta.getFieldNullIf()[i], meta.getFieldIfNull()[i], meta.getFieldTrimType()[i]);
        } catch (Exception e) {
            throw new KettleValueException(BaseMessages.getString(PKG, "FieldSplitter.Log.ErrorConvertingSplitValue", rawValue, meta.getSplitField() + "]!"), e);
        }
        outputRow[data.fieldnr + i] = value;
    }
    return outputRow;
}

Example 20 with KettleValueException

use of org.pentaho.di.core.exception.KettleValueException in project pentaho-kettle by pentaho.

the class HTTP method constructUrlBuilder.

private URIBuilder constructUrlBuilder(RowMetaInterface outputRowMeta, Object[] row) throws KettleValueException, KettleException {
    URIBuilder uriBuilder;
    try {
        String baseUrl = data.realUrl;
        if (meta.isUrlInField()) {
            // get dynamic url
            baseUrl = outputRowMeta.getString(row, data.indexOfUrlField);
        }
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "HTTP.Log.Connecting", baseUrl));
        }
        // the base URL with variable substitution
        uriBuilder = new URIBuilder(baseUrl);
        List<NameValuePair> queryParams = uriBuilder.getQueryParams();
        for (int i = 0; i < data.argnrs.length; i++) {
            String key = meta.getArgumentParameter()[i];
            String value = outputRowMeta.getString(row, data.argnrs[i]);
            BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, value);
            queryParams.add(basicNameValuePair);
        }
        if (!queryParams.isEmpty()) {
            uriBuilder.setParameters(queryParams);
        }
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "HTTP.Log.UnableCreateUrl"), e);
    }
    return uriBuilder;
}