Examples with KettleStepException org.pentaho.di.core.exception.KettleStepException used on opensource projects

Example 31 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class BaseStep method specialPartitioning.

private void specialPartitioning(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
    if (nextStepPartitioningMeta == null) {
        // Look up the partitioning of the next step.
        // This is the case for non-clustered partitioning...
        // 
        List<StepMeta> nextSteps = transMeta.findNextSteps(stepMeta);
        if (nextSteps.size() > 0) {
            nextStepPartitioningMeta = nextSteps.get(0).getStepPartitioningMeta();
        }
    // TODO: throw exception if we're not partitioning yet.
    // For now it throws a NP Exception.
    }
    int partitionNr;
    try {
        partitionNr = nextStepPartitioningMeta.getPartition(rowMeta, row);
    } catch (KettleException e) {
        throw new KettleStepException("Unable to convert a value to integer while calculating the partition number", e);
    }
    RowSet selectedRowSet = null;
    if (clusteredPartitioningFirst) {
        clusteredPartitioningFirst = false;
        // We are only running remotely if both the distribution is there AND if the distribution is actually contains
        // something.
        // 
        clusteredPartitioning = transMeta.getSlaveStepCopyPartitionDistribution() != null && !transMeta.getSlaveStepCopyPartitionDistribution().getDistribution().isEmpty();
    }
    // 
    if (clusteredPartitioning) {
        // 
        if (partitionNrRowSetList == null) {
            partitionNrRowSetList = new RowSet[outputRowSets.size()];
            // The distribution is calculated during transformation split
            // The slave-step-copy distribution is passed onto the slave transformation
            // 
            SlaveStepCopyPartitionDistribution distribution = transMeta.getSlaveStepCopyPartitionDistribution();
            String nextPartitionSchemaName = TransSplitter.createPartitionSchemaNameFromTarget(nextStepPartitioningMeta.getPartitionSchema().getName());
            for (RowSet outputRowSet : outputRowSets) {
                try {
                    // Look at the pre-determined distribution, decided at "transformation split" time.
                    // 
                    int partNr = distribution.getPartition(outputRowSet.getRemoteSlaveServerName(), nextPartitionSchemaName, outputRowSet.getDestinationStepCopy());
                    if (partNr < 0) {
                        throw new KettleStepException("Unable to find partition using rowset data, slave=" + outputRowSet.getRemoteSlaveServerName() + ", partition schema=" + nextStepPartitioningMeta.getPartitionSchema().getName() + ", copy=" + outputRowSet.getDestinationStepCopy());
                    }
                    partitionNrRowSetList[partNr] = outputRowSet;
                } catch (NullPointerException e) {
                    throw (e);
                }
            }
        }
        // 
        if (partitionNr < partitionNrRowSetList.length) {
            selectedRowSet = partitionNrRowSetList[partitionNr];
        } else {
            String rowsets = "";
            for (RowSet rowSet : partitionNrRowSetList) {
                rowsets += "[" + rowSet.toString() + "] ";
            }
            throw new KettleStepException("Internal error: the referenced partition nr '" + partitionNr + "' is higher than the maximum of '" + (partitionNrRowSetList.length - 1) + ".  The available row sets are: {" + rowsets + "}");
        }
        if (selectedRowSet == null) {
            logBasic(BaseMessages.getString(PKG, "BaseStep.TargetRowsetIsNotAvailable", partitionNr));
        } else {
            // Wait
            putRowToRowSet(selectedRowSet, rowMeta, row);
            incrementLinesWritten();
            if (log.isRowLevel()) {
                try {
                    logRowlevel("Partitioned #" + partitionNr + " to " + selectedRowSet + ", row=" + rowMeta.getString(row));
                } catch (KettleValueException e) {
                    throw new KettleStepException(e);
                }
            }
        }
    } else {
        // Local partitioning...
        // Put the row forward to the next step according to the partition rule.
        // 
        // Count of partitioned row at one step
        int partCount = ((BasePartitioner) nextStepPartitioningMeta.getPartitioner()).getNrPartitions();
        for (int i = 0; i < nextSteps.length; i++) {
            selectedRowSet = outputRowSets.get(partitionNr + i * partCount);
            if (selectedRowSet == null) {
                logBasic(BaseMessages.getString(PKG, "BaseStep.TargetRowsetIsNotAvailable", partitionNr));
            } else {
                // Wait
                putRowToRowSet(selectedRowSet, rowMeta, row);
                incrementLinesWritten();
                if (log.isRowLevel()) {
                    try {
                        logRowlevel(BaseMessages.getString(PKG, "BaseStep.PartitionedToRow", partitionNr, selectedRowSet, rowMeta.getString(row)));
                    } catch (KettleValueException e) {
                        throw new KettleStepException(e);
                    }
                }
            }
        }
    }
}

Example 32 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class BaseStep method noPartitioning.

private void noPartitioning(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
    if (distributed) {
        if (rowDistribution != null) {
            // Plugin defined row distribution!
            // 
            rowDistribution.distributeRow(rowMeta, row, this);
            incrementLinesWritten();
        } else {
            // ROUND ROBIN DISTRIBUTION:
            // --------------------------
            // Copy the row to the "next" output rowset.
            // We keep the next one in out_handling
            // 
            RowSet rs = outputRowSets.get(currentOutputRowSetNr);
            // 
            if (isUsingThreadPriorityManagment() && !rs.isDone() && rs.size() >= upperBufferBoundary && !isStopped()) {
                try {
                    Thread.sleep(0, 1);
                } catch (InterruptedException e) {
                // Ignore sleep interruption exception
                }
            }
            // Loop until we find room in the target rowset
            // 
            putRowToRowSet(rs, rowMeta, row);
            incrementLinesWritten();
            // 
            if (outputRowSets.size() > 1) {
                currentOutputRowSetNr++;
                if (currentOutputRowSetNr >= outputRowSets.size()) {
                    currentOutputRowSetNr = 0;
                }
            }
        }
    } else {
        // Copy to the row in the other output rowsets...
        for (int i = 1; i < outputRowSets.size(); i++) {
            // start at 1
            RowSet rs = outputRowSets.get(i);
            // 
            if (isUsingThreadPriorityManagment() && !rs.isDone() && rs.size() >= upperBufferBoundary && !isStopped()) {
                try {
                    Thread.sleep(0, 1);
                } catch (InterruptedException e) {
                // Ignore sleep interruption exception
                }
            }
            try {
                // Loop until we find room in the target rowset
                // 
                putRowToRowSet(rs, rowMeta, rowMeta.cloneRow(row));
                incrementLinesWritten();
            } catch (KettleValueException e) {
                throw new KettleStepException("Unable to clone row while copying rows to multiple target steps", e);
            }
        }
        // set row in first output rowset
        // 
        RowSet rs = outputRowSets.get(0);
        putRowToRowSet(rs, rowMeta, row);
        incrementLinesWritten();
    }
}

Example 33 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class FieldSplitterMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // Remove the field to split
    int idx = r.indexOfValue(getSplitField());
    if (idx < 0) {
        // not found
        throw new RuntimeException(BaseMessages.getString(PKG, "FieldSplitter.Log.CouldNotFindFieldToSplit", getSplitField()));
    }
    // Add the new fields at the place of the index --> replace!
    int count = getFieldsCount();
    for (int i = 0; i < count; i++) {
        try {
            final ValueMetaInterface v = ValueMetaFactory.createValueMeta(getFieldName()[i], getFieldType()[i]);
            v.setLength(getFieldLength()[i], getFieldPrecision()[i]);
            v.setOrigin(name);
            v.setConversionMask(getFieldFormat()[i]);
            v.setDecimalSymbol(getFieldDecimal()[i]);
            v.setGroupingSymbol(getFieldGroup()[i]);
            v.setCurrencySymbol(getFieldCurrency()[i]);
            v.setTrimType(getFieldTrimType()[i]);
            // v.setDateFormatLocale(dateFormatLocale);
            if (i == 0 && idx >= 0) {
                // the first valueMeta (splitField) will be replaced
                r.setValueMeta(idx, v);
            } else {
                // other valueMeta will be added
                if (idx >= r.size()) {
                    r.addValueMeta(v);
                }
                r.addValueMeta(idx + i, v);
            }
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
}

Example 34 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class TextFileInputMeta method getFields.

@Override
public void getFields(RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    if (!inputFiles.passingThruFields) {
        // all incoming fields are not transmitted !
        row.clear();
    } else {
        if (info != null) {
            boolean found = false;
            for (int i = 0; i < info.length && !found; i++) {
                if (info[i] != null) {
                    row.mergeRowMeta(info[i], name);
                    found = true;
                }
            }
        }
    }
    for (int i = 0; i < inputFields.length; i++) {
        BaseFileField field = inputFields[i];
        int type = field.getType();
        if (type == ValueMetaInterface.TYPE_NONE) {
            type = ValueMetaInterface.TYPE_STRING;
        }
        try {
            ValueMetaInterface v = ValueMetaFactory.createValueMeta(field.getName(), type);
            v.setLength(field.getLength());
            v.setPrecision(field.getPrecision());
            v.setOrigin(name);
            v.setConversionMask(field.getFormat());
            v.setDecimalSymbol(field.getDecimalSymbol());
            v.setGroupingSymbol(field.getGroupSymbol());
            v.setCurrencySymbol(field.getCurrencySymbol());
            v.setDateFormatLenient(content.dateFormatLenient);
            v.setDateFormatLocale(content.dateFormatLocale);
            v.setTrimType(field.getTrimType());
            row.addValueMeta(v);
        } catch (Exception e) {
            throw new KettleStepException(e);
        }
    }
    if (errorHandling.errorIgnored) {
        if (errorCountField != null && errorCountField.length() > 0) {
            ValueMetaInterface v = new ValueMetaInteger(errorCountField);
            v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
        if (errorFieldsField != null && errorFieldsField.length() > 0) {
            ValueMetaInterface v = new ValueMetaString(errorFieldsField);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
        if (errorTextField != null && errorTextField.length() > 0) {
            ValueMetaInterface v = new ValueMetaString(errorTextField);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
    }
    if (content.includeFilename) {
        ValueMetaInterface v = new ValueMetaString(content.filenameField);
        v.setLength(100);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (content.includeRowNumber) {
        ValueMetaInterface v = new ValueMetaInteger(content.rowNumberField);
        v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.shortFilenameField)) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(additionalOutputFields.shortFilenameField));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.extensionField)) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(additionalOutputFields.extensionField));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.pathField)) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(additionalOutputFields.pathField));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.sizeField)) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(additionalOutputFields.sizeField));
        v.setOrigin(name);
        v.setLength(9);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.hiddenField)) {
        ValueMetaInterface v = new ValueMetaBoolean(space.environmentSubstitute(additionalOutputFields.hiddenField));
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.lastModificationField)) {
        ValueMetaInterface v = new ValueMetaDate(space.environmentSubstitute(additionalOutputFields.lastModificationField));
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.uriField)) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(additionalOutputFields.uriField));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (StringUtils.isNotBlank(additionalOutputFields.rootUriField)) {
        ValueMetaInterface v = new ValueMetaString(additionalOutputFields.rootUriField);
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
}

Example 35 with KettleStepException

use of org.pentaho.di.core.exception.KettleStepException in project pentaho-kettle by pentaho.

the class FilesFromResultMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // Add the fields from a ResultFile
    try {
        ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject("foo.bar", space), "parentOrigin", "origin");
        RowMetaAndData add = resultFile.getRow();
        // Set the origin on the fields...
        for (int i = 0; i < add.size(); i++) {
            add.getValueMeta(i).setOrigin(name);
        }
        r.addRowMeta(add.getRowMeta());
    } catch (KettleFileException e) {
        throw new KettleStepException(e);
    }
}