Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 51 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class Validator method readSourceValuesFromInfoSteps.

void readSourceValuesFromInfoSteps() throws KettleStepException {
    Map<String, Integer> inputStepWasProcessed = new HashMap<>();
    for (int i = 0; i < meta.getValidations().size(); i++) {
        Validation field = meta.getValidations().get(i);
        List<StreamInterface> streams = meta.getStepIOMeta().getInfoStreams();
        // 
        if (field.isSourcingValues()) {
            if (streams.get(i).getStepMeta() == null) {
                throw new KettleStepException("There is no valid source step specified for the allowed values of validation [" + field.getName() + "]");
            }
            if (Utils.isEmpty(field.getSourcingField())) {
                throw new KettleStepException("There is no valid source field specified for the allowed values of validation [" + field.getName() + "]");
            }
            // Still here : OK, read the data from the specified step...
            // The data is stored in data.listValues[i] and data.constantsMeta
            // 
            String stepName = streams.get(i).getStepname();
            if (inputStepWasProcessed.containsKey(stepName)) {
                // step was processed for other StreamInterface
                data.listValues[i] = data.listValues[inputStepWasProcessed.get(stepName)];
                data.constantsMeta[i] = data.constantsMeta[inputStepWasProcessed.get(stepName)];
                continue;
            }
            RowSet allowedRowSet = findInputRowSet(stepName);
            int fieldIndex = -1;
            List<Object> allowedValues = new ArrayList<Object>();
            Object[] allowedRowData = getRowFrom(allowedRowSet);
            while (allowedRowData != null) {
                RowMetaInterface allowedRowMeta = allowedRowSet.getRowMeta();
                if (fieldIndex < 0) {
                    fieldIndex = allowedRowMeta.indexOfValue(field.getSourcingField());
                    if (fieldIndex < 0) {
                        throw new KettleStepException("Source field [" + field.getSourcingField() + "] is not found in the source row data");
                    }
                    data.constantsMeta[i] = allowedRowMeta.getValueMeta(fieldIndex);
                }
                Object allowedValue = allowedRowData[fieldIndex];
                if (allowedValue != null) {
                    allowedValues.add(allowedValue);
                }
                // Grab another row too...
                // 
                allowedRowData = getRowFrom(allowedRowSet);
            }
            // Set the list values in the data block...
            // 
            data.listValues[i] = allowedValues.toArray(new Object[allowedValues.size()]);
            inputStepWasProcessed.put(stepName, i);
        }
    }
}

Example 52 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class StreamLookup method readLookupValues.

private boolean readLookupValues() throws KettleException {
    data.infoStream = meta.getStepIOMeta().getInfoStreams().get(0);
    if (data.infoStream.getStepMeta() == null) {
        logError(BaseMessages.getString(PKG, "StreamLookup.Log.NoLookupStepSpecified"));
        return false;
    }
    if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "StreamLookup.Log.ReadingFromStream") + data.infoStream.getStepname() + "]");
    }
    int[] keyNrs = new int[meta.getKeylookup().length];
    int[] valueNrs = new int[meta.getValue().length];
    boolean firstRun = true;
    // Which row set do we read from?
    // 
    RowSet rowSet = findInputRowSet(data.infoStream.getStepname());
    // rows are originating from "lookup_from"
    Object[] rowData = getRowFrom(rowSet);
    while (rowData != null) {
        if (log.isRowLevel()) {
            logRowlevel(BaseMessages.getString(PKG, "StreamLookup.Log.ReadLookupRow") + rowSet.getRowMeta().getString(rowData));
        }
        if (firstRun) {
            firstRun = false;
            data.hasLookupRows = true;
            data.infoMeta = rowSet.getRowMeta().clone();
            RowMetaInterface cacheKeyMeta = new RowMeta();
            RowMetaInterface cacheValueMeta = new RowMeta();
            // Look up the keys in the source rows
            for (int i = 0; i < meta.getKeylookup().length; i++) {
                keyNrs[i] = rowSet.getRowMeta().indexOfValue(meta.getKeylookup()[i]);
                if (keyNrs[i] < 0) {
                    throw new KettleStepException(BaseMessages.getString(PKG, "StreamLookup.Exception.UnableToFindField", meta.getKeylookup()[i]));
                }
                cacheKeyMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(keyNrs[i]));
            }
            // Save the data types of the keys to optionally convert input rows later on...
            if (data.keyTypes == null) {
                data.keyTypes = cacheKeyMeta.clone();
            }
            // Cache keys are stored as normal types, not binary
            for (int i = 0; i < keyNrs.length; i++) {
                cacheKeyMeta.getValueMeta(i).setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            }
            for (int v = 0; v < meta.getValue().length; v++) {
                valueNrs[v] = rowSet.getRowMeta().indexOfValue(meta.getValue()[v]);
                if (valueNrs[v] < 0) {
                    throw new KettleStepException(BaseMessages.getString(PKG, "StreamLookup.Exception.UnableToFindField", meta.getValue()[v]));
                }
                cacheValueMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(valueNrs[v]));
            }
            data.cacheKeyMeta = cacheKeyMeta;
            data.cacheValueMeta = cacheValueMeta;
        }
        Object[] keyData = new Object[keyNrs.length];
        for (int i = 0; i < keyNrs.length; i++) {
            ValueMetaInterface keyMeta = data.keyTypes.getValueMeta(i);
            // Convert keys to normal storage type
            keyData[i] = keyMeta.convertToNormalStorageType(rowData[keyNrs[i]]);
        }
        Object[] valueData = new Object[valueNrs.length];
        for (int i = 0; i < valueNrs.length; i++) {
            // Store value as is, avoid preliminary binary->normal storage type conversion
            valueData[i] = rowData[valueNrs[i]];
        }
        addToCache(data.cacheKeyMeta, keyData, data.cacheValueMeta, valueData);
        rowData = getRowFrom(rowSet);
    }
    return true;
}

Example 53 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-cassandra-plugin by pentaho.

the class SSTableOutputIT method testCQLS2SSTableWriter.

@Test
public void testCQLS2SSTableWriter() throws Exception {
    SSTableOutput ssTableOutput = new SSTableOutput(helper.stepMeta, helper.stepDataInterface, 0, helper.transMeta, helper.trans);
    ValueMetaInterface one = new ValueMetaBase("key", ValueMetaBase.TYPE_INTEGER);
    ValueMetaInterface two = new ValueMetaBase("two", ValueMetaBase.TYPE_STRING);
    List<ValueMetaInterface> valueMetaList = new ArrayList<ValueMetaInterface>();
    valueMetaList.add(one);
    valueMetaList.add(two);
    String[] fieldNames = new String[] { "key", "two" };
    RowMetaInterface inputRowMeta = mock(RowMetaInterface.class);
    when(inputRowMeta.clone()).thenReturn(inputRowMeta);
    when(inputRowMeta.size()).thenReturn(2);
    when(inputRowMeta.getFieldNames()).thenReturn(fieldNames);
    when(inputRowMeta.getValueMetaList()).thenReturn(valueMetaList);
    RowSet rowset = helper.getMockInputRowSet(new Object[] { 1, "some" });
    when(rowset.getRowMeta()).thenReturn(inputRowMeta);
    ssTableOutput.addRowSetToInputRowSets(rowset);
    SSTableOutputMeta meta = createStepMeta(false);
    ssTableOutput.init(meta, helper.initStepDataInterface);
    ssTableOutput.processRow(meta, helper.processRowsStepDataInterface);
    Assert.assertEquals("Step init error.", 0, ssTableOutput.getErrors());
    assertEquals("org.pentaho.di.trans.steps.cassandrasstableoutput.writer.CQL2SSTableWriter", ssTableOutput.writer.getClass().getName());
    ssTableOutput.dispose(meta, helper.initStepDataInterface);
    Assert.assertEquals("Step dispose error", 0, ssTableOutput.getErrors());
}

Example 54 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-cassandra-plugin by pentaho.

the class SSTableOutputIT method testCQLS3SSTableWriter.

@Test
public void testCQLS3SSTableWriter() throws Exception {
    SSTableOutput ssTableOutput = new SSTableOutput(helper.stepMeta, helper.stepDataInterface, 0, helper.transMeta, helper.trans);
    i = new AtomicInteger(0);
    ValueMetaInterface one = new ValueMetaBase("key", ValueMetaBase.TYPE_INTEGER);
    ValueMetaInterface two = new ValueMetaBase("two", ValueMetaBase.TYPE_STRING);
    List<ValueMetaInterface> valueMetaList = new ArrayList<ValueMetaInterface>();
    valueMetaList.add(one);
    valueMetaList.add(two);
    String[] fieldNames = new String[] { "key", "two" };
    RowMetaInterface inputRowMeta = mock(RowMetaInterface.class);
    when(inputRowMeta.clone()).thenReturn(inputRowMeta);
    when(inputRowMeta.size()).thenReturn(2);
    when(inputRowMeta.getFieldNames()).thenReturn(fieldNames);
    when(inputRowMeta.getValueMetaList()).thenReturn(valueMetaList);
    when(inputRowMeta.indexOfValue(anyString())).thenAnswer(new Answer<Integer>() {

        @Override
        public Integer answer(InvocationOnMock invocation) throws Throwable {
            return i.getAndIncrement();
        }
    });
    RowSet rowset = helper.getMockInputRowSet(new Object[] { 1L, "some" });
    when(rowset.getRowMeta()).thenReturn(inputRowMeta);
    ssTableOutput.addRowSetToInputRowSets(rowset);
    SSTableOutputMeta meta = createStepMeta(true);
    ssTableOutput.init(meta, helper.initStepDataInterface);
    ssTableOutput.processRow(meta, helper.processRowsStepDataInterface);
    Assert.assertEquals("Step init error.", 0, ssTableOutput.getErrors());
    assertEquals("org.pentaho.di.trans.steps.cassandrasstableoutput.writer.CQL3SSTableWriter", ssTableOutput.writer.getClass().getName());
    ssTableOutput.dispose(meta, helper.initStepDataInterface);
    Assert.assertEquals("Step dispose error", 0, ssTableOutput.getErrors());
}

Example 55 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class XBaseInput method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (XBaseInputMeta) smi;
    data = (XBaseInputData) sdi;
    // See if we need to get a list of files from input...
    if (first) {
        // we just got started
        first = false;
        // The output row meta data, what does it look like?
        // 
        data.outputRowMeta = new RowMeta();
        if (meta.isAcceptingFilenames()) {
            // Read the files from the specified input stream...
            data.files.getFiles().clear();
            int idx = -1;
            RowSet rowSet = findInputRowSet(meta.getAcceptingStepName());
            Object[] fileRowData = getRowFrom(rowSet);
            while (fileRowData != null) {
                RowMetaInterface fileRowMeta = rowSet.getRowMeta();
                if (idx < 0) {
                    idx = fileRowMeta.indexOfValue(meta.getAcceptingField());
                    if (idx < 0) {
                        logError(BaseMessages.getString(PKG, "XBaseInput.Log.Error.UnableToFindFilenameField", meta.getAcceptingField()));
                        setErrors(1);
                        stopAll();
                        return false;
                    }
                }
                try {
                    String filename = fileRowMeta.getString(fileRowData, idx);
                    data.files.addFile(KettleVFS.getFileObject(filename, getTransMeta()));
                } catch (Exception e) {
                    throw new KettleException(e);
                }
                // Grab another row
                // 
                fileRowData = getRowFrom(rowSet);
            }
            if (data.files.nrOfFiles() == 0) {
                logBasic(BaseMessages.getString(PKG, "XBaseInput.Log.Error.NoFilesSpecified"));
                setOutputDone();
                return false;
            }
        }
        data.outputRowMeta = meta.getOutputFields(data.files, getStepname());
        // Open the first file & read the required rows in the buffer, stop
        // if it fails, exception will stop processLoop
        // 
        openNextFile();
    }
    // Allocate the output row in advance, because we possibly want to add a few extra fields...
    // 
    Object[] row = data.xbi.getRow(RowDataUtil.allocateRowData(data.outputRowMeta.size()));
    while (row == null && data.fileNr < data.files.nrOfFiles()) {
        // No more rows left in this file
        openNextFile();
        row = data.xbi.getRow(RowDataUtil.allocateRowData(data.outputRowMeta.size()));
    }
    if (row == null) {
        // signal end to receiver(s)
        setOutputDone();
        // end of data or error.
        return false;
    }
    // OK, so we have read a line: increment the input counter
    incrementLinesInput();
    int outputIndex = data.fields.size();
    // Possibly add a filename...
    if (meta.includeFilename()) {
        row[outputIndex++] = data.file_dbf.getName().getURI();
    }
    // Possibly add a row number...
    if (meta.isRowNrAdded()) {
        row[outputIndex++] = new Long(getLinesInput());
    }
    // fill the rowset(s). (wait for empty)
    putRow(data.outputRowMeta, row);
    if (checkFeedback(getLinesInput())) {
        logBasic(BaseMessages.getString(PKG, "XBaseInput.Log.LineNr") + getLinesInput());
    }
    if (meta.getRowLimit() > 0 && getLinesInput() >= meta.getRowLimit()) {
        // limit has been reached: stop now.
        setOutputDone();
        return false;
    }
    return true;
}