Examples with RowSet org.pentaho.di.core.RowSet used on opensource projects

Example 86 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class AddXMLTest method createSourceRowSet.

private RowSet createSourceRowSet(String source) {
    RowSet sourceRowSet = stepMockHelper.getMockInputRowSet(new String[] { source });
    RowMetaInterface sourceRowMeta = mock(RowMetaInterface.class);
    when(sourceRowMeta.getFieldNames()).thenReturn(new String[] { source });
    when(sourceRowSet.getRowMeta()).thenReturn(sourceRowMeta);
    return sourceRowSet;
}

Example 87 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class TextFileInputDialogTest method testMinimalWidth_PDI_14253.

@Test
public void testMinimalWidth_PDI_14253() throws Exception {
    final String virtualFile = "ram://pdi-14253.txt";
    KettleVFS.getFileObject(virtualFile).createFile();
    final String content = "r1c1,  r1c2\nr2c1  ,  r2c2  ";
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    bos.write(content.getBytes());
    OutputStream os = KettleVFS.getFileObject(virtualFile).getContent().getOutputStream();
    IOUtils.copy(new ByteArrayInputStream(bos.toByteArray()), os);
    os.close();
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.content.lineWrapped = false;
    meta.inputFields = new BaseFileField[] { new BaseFileField("col1", -1, -1), new BaseFileField("col2", -1, -1) };
    meta.content.fileCompression = "None";
    meta.content.fileType = "CSV";
    meta.content.header = false;
    meta.content.nrHeaderLines = -1;
    meta.content.footer = false;
    meta.content.nrFooterLines = -1;
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables() {
    });
    data.filePlayList = new FilePlayListAll();
    TextFileInputDialog dialog = new TextFileInputDialog(mock(Shell.class), meta, mock(TransMeta.class), "TFIMinimalWidthTest");
    TableView tv = mock(TableView.class);
    when(tv.nrNonEmpty()).thenReturn(0);
    // click the Minimal width button
    dialog.setMinimalWidth(tv);
    RowSet output = new BlockingRowSet(5);
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    input.setOutputRowSets(Collections.singletonList(output));
    while (input.processRow(meta, data)) {
    // wait until the step completes executing
    }
    Object[] row1 = output.getRowImmediate();
    assertRow(row1, "r1c1", "r1c2");
    Object[] row2 = output.getRowImmediate();
    assertRow(row2, "r2c1", "r2c2");
    KettleVFS.getFileObject(virtualFile).delete();
}

Example 88 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class MetaInject method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MetaInjectMeta) smi;
    data = (MetaInjectData) sdi;
    // Read the data from all input steps and keep it in memory...
    // Skip the step from which we stream data. Keep that available for runtime action.
    // 
    data.rowMap = new HashMap<String, List<RowMetaAndData>>();
    for (String prevStepName : getTransMeta().getPrevStepNames(getStepMeta())) {
        // 
        if (!data.streaming || !prevStepName.equalsIgnoreCase(data.streamingSourceStepname)) {
            List<RowMetaAndData> list = new ArrayList<RowMetaAndData>();
            RowSet rowSet = findInputRowSet(prevStepName);
            Object[] row = getRowFrom(rowSet);
            while (row != null) {
                RowMetaAndData rd = new RowMetaAndData();
                rd.setRowMeta(rowSet.getRowMeta());
                rd.setData(row);
                list.add(rd);
                row = getRowFrom(rowSet);
            }
            if (!list.isEmpty()) {
                data.rowMap.put(prevStepName, list);
            }
        }
    }
    List<StepMeta> steps = data.transMeta.getSteps();
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjection(en.getKey(), en.getValue());
    }
    /*
     * constants injection should be executed after steps, because if constant should be inserted into target with array
     * in path, constants should be inserted into all arrays items
     */
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjectionConstants(en.getKey(), en.getValue());
    }
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        en.getValue().searchInfoAndTargetSteps(steps);
    }
    for (String targetStepName : data.stepInjectionMap.keySet()) {
        if (!data.stepInjectionMetasMap.containsKey(targetStepName)) {
            oldInjection(targetStepName);
            StepMeta targetStep = StepMeta.findStep(steps, targetStepName);
            if (targetStep != null) {
                targetStep.getStepMetaInterface().searchInfoAndTargetSteps(steps);
            }
        }
    }
    if (!meta.isNoExecution()) {
        // Now we can execute this modified transformation metadata.
        // 
        final Trans injectTrans = createInjectTrans();
        injectTrans.setMetaStore(getMetaStore());
        if (getTrans().getParentJob() != null) {
            // See PDI-13224
            injectTrans.setParentJob(getTrans().getParentJob());
        }
        getTrans().addTransStoppedListener(new TransStoppedListener() {

            public void transStopped(Trans parentTrans) {
                injectTrans.stopAll();
            }
        });
        injectTrans.prepareExecution(null);
        // See if we need to stream some data over...
        // 
        RowProducer rowProducer = null;
        if (data.streaming) {
            rowProducer = injectTrans.addRowProducer(data.streamingTargetStepname, 0);
        }
        // Finally, add the mapping transformation to the active sub-transformations
        // map in the parent transformation
        // 
        getTrans().addActiveSubTransformation(getStepname(), injectTrans);
        if (!Utils.isEmpty(meta.getSourceStepName())) {
            StepInterface stepInterface = injectTrans.getStepInterface(meta.getSourceStepName(), 0);
            if (stepInterface == null) {
                throw new KettleException("Unable to find step '" + meta.getSourceStepName() + "' to read from.");
            }
            stepInterface.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    // Just pass along the data as output of this step...
                    // 
                    MetaInject.this.putRow(rowMeta, row);
                }
            });
        }
        injectTrans.startThreads();
        if (data.streaming) {
            // Deplete all the rows from the parent transformation into the modified transformation
            // 
            RowSet rowSet = findInputRowSet(data.streamingSourceStepname);
            if (rowSet == null) {
                throw new KettleException("Unable to find step '" + data.streamingSourceStepname + "' to stream data from");
            }
            Object[] row = getRowFrom(rowSet);
            while (row != null && !isStopped()) {
                rowProducer.putRow(rowSet.getRowMeta(), row);
                row = getRowFrom(rowSet);
            }
            rowProducer.finished();
        }
        // 
        while (!injectTrans.isFinished() && !injectTrans.isStopped() && !isStopped()) {
            copyResult(injectTrans);
            // Wait a little bit.
            try {
                Thread.sleep(50);
            } catch (Exception e) {
            // Ignore errors
            }
        }
        copyResult(injectTrans);
        waitUntilFinished(injectTrans);
    }
    // let the transformation complete it's execution to allow for any customizations to MDI to happen in the init methods of steps
    if (log.isDetailed()) {
        logDetailed("XML of transformation after injection: " + data.transMeta.getXML());
    }
    String targetFile = environmentSubstitute(meta.getTargetFile());
    if (!Utils.isEmpty(targetFile)) {
        writeInjectedKtr(targetFile);
    }
    // All done!
    setOutputDone();
    return false;
}

Example 89 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class FastJsonReader method getEmptyResponse.

private RowSet getEmptyResponse() {
    RowSet nullInputResponse = new SingleRowRowSet();
    nullInputResponse.putRow(null, new Object[fields.length]);
    nullInputResponse.setDone();
    return nullInputResponse;
}

Example 90 with RowSet

use of org.pentaho.di.core.RowSet in project pentaho-kettle by pentaho.

the class AggregateRowsTest method testProcessRow.

@Test
public void testProcessRow() throws KettleException {
    AggregateRows aggregateRows = new AggregateRows(stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    aggregateRows.init(stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface);
    aggregateRows.setInputRowSets(new ArrayList<RowSet>(Arrays.asList(createSourceRowSet("TEST"))));
    int fieldSize = stepMockHelper.initStepMetaInterface.getFieldName().length;
    AggregateRowsData data = new AggregateRowsData();
    data.fieldnrs = new int[fieldSize];
    data.counts = new long[fieldSize];
    data.values = new Object[fieldSize];
    assertTrue(aggregateRows.processRow(stepMockHelper.initStepMetaInterface, data));
    assertTrue(aggregateRows.getErrors() == 0);
    assertTrue(aggregateRows.getLinesRead() > 0);
    RowMetaInterface outputRowMeta = mock(RowMetaInterface.class);
    when(outputRowMeta.size()).thenReturn(fieldSize);
    data.outputRowMeta = outputRowMeta;
    assertFalse(aggregateRows.processRow(stepMockHelper.initStepMetaInterface, data));
    assertTrue(aggregateRows.getLinesWritten() > 0);
}