Examples with RowAdapter org.pentaho.di.trans.step.RowAdapter used on opensource projects

Example 11 with RowAdapter

use of org.pentaho.di.trans.step.RowAdapter in project pdi-dataservice-server-plugin by pentaho.

the class ServiceObserverTest method verifyCachedRowIsCloned.

@Test
public void verifyCachedRowIsCloned() throws Exception {
    when(stepInterface.isStopped()).thenReturn(false);
    observer.run();
    verify(stepInterface).addRowListener(rowAdapterCaptor.capture());
    verify(stepInterface).addStepListener(stepAdapterCaptor.capture());
    RowAdapter rowAdapter = rowAdapterCaptor.getValue();
    StepAdapter stepAdapter = stepAdapterCaptor.getValue();
    Object[] clonedRow = new Object[0];
    when(rowMeta.cloneRow(row)).thenReturn(clonedRow);
    rowAdapter.rowWrittenEvent(rowMeta, row);
    verify(rowMeta).cloneRow(row);
    stepAdapter.stepFinished(null, null, stepInterface);
    CachedService cachedService = observer.get();
    assertThat(cachedService.getRowMetaAndData().get(0).getData(), is(clonedRow));
}

Example 12 with RowAdapter

use of org.pentaho.di.trans.step.RowAdapter in project pentaho-kettle by pentaho.

the class SubtransExecutor method execute.

public Optional<Result> execute(List<RowMetaAndData> rows) throws KettleException {
    if (rows.isEmpty() || stopped) {
        return Optional.empty();
    }
    Trans subtrans = this.createSubtrans();
    running.add(subtrans);
    parentTrans.addActiveSubTransformation(subTransName, subtrans);
    // Pass parameter values
    passParametersToTrans(subtrans, rows.get(0));
    Result result = new Result();
    result.setRows(rows);
    subtrans.setPreviousResult(result);
    subtrans.prepareExecution(this.parentTrans.getArguments());
    List<RowMetaAndData> rowMetaAndData = new ArrayList<>();
    subtrans.getSteps().stream().filter(c -> c.step.getStepname().equalsIgnoreCase(subStep)).findFirst().ifPresent(c -> c.step.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) {
            rowMetaAndData.add(new RowMetaAndData(rowMeta, row));
        }
    }));
    subtrans.startThreads();
    subtrans.waitUntilFinished();
    updateStatuses(subtrans);
    running.remove(subtrans);
    Result subtransResult = subtrans.getResult();
    subtransResult.setRows(rowMetaAndData);
    return Optional.of(subtransResult);
}

Example 13 with RowAdapter

use of org.pentaho.di.trans.step.RowAdapter in project pentaho-kettle by pentaho.

the class TransPreviewDelegate method capturePreviewData.

public void capturePreviewData(final Trans trans, List<StepMeta> stepMetas) {
    final StringBuffer loggingText = new StringBuffer();
    // First clean out previous preview data. Otherwise this method leaks memory like crazy.
    // 
    previewLogMap.clear();
    previewMetaMap.clear();
    previewDataMap.clear();
    try {
        final TransMeta transMeta = trans.getTransMeta();
        for (final StepMeta stepMeta : stepMetas) {
            final RowMetaInterface rowMeta = transMeta.getStepFields(stepMeta).clone();
            previewMetaMap.put(stepMeta, rowMeta);
            final List<Object[]> rowsData;
            if (previewMode == PreviewMode.LAST) {
                rowsData = new LinkedList<Object[]>();
            } else {
                rowsData = new ArrayList<Object[]>();
            }
            previewDataMap.put(stepMeta, rowsData);
            previewLogMap.put(stepMeta, loggingText);
            StepInterface step = trans.findRunThread(stepMeta.getName());
            if (step != null) {
                switch(previewMode) {
                    case LAST:
                        step.addRowListener(new RowAdapter() {

                            @Override
                            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                                try {
                                    rowsData.add(rowMeta.cloneRow(row));
                                    if (rowsData.size() > PropsUI.getInstance().getDefaultPreviewSize()) {
                                        rowsData.remove(0);
                                    }
                                } catch (Exception e) {
                                    throw new KettleStepException("Unable to clone row for metadata : " + rowMeta, e);
                                }
                            }
                        });
                        break;
                    default:
                        step.addRowListener(new RowAdapter() {

                            @Override
                            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                                if (rowsData.size() < PropsUI.getInstance().getDefaultPreviewSize()) {
                                    try {
                                        rowsData.add(rowMeta.cloneRow(row));
                                    } catch (Exception e) {
                                        throw new KettleStepException("Unable to clone row for metadata : " + rowMeta, e);
                                    }
                                }
                            }
                        });
                        break;
                }
            }
        }
    } catch (Exception e) {
        loggingText.append(Const.getStackTracker(e));
    }
    // In case there were errors during preview...
    // 
    trans.addTransListener(new TransAdapter() {

        @Override
        public void transFinished(Trans trans) throws KettleException {
            // 
            if (trans.getErrors() != 0) {
                // 
                for (StepMetaDataCombi combi : trans.getSteps()) {
                    if (combi.copy == 0) {
                        StringBuffer logBuffer = KettleLogStore.getAppender().getBuffer(combi.step.getLogChannel().getLogChannelId(), false);
                        previewLogMap.put(combi.stepMeta, logBuffer);
                    }
                }
            }
        }
    });
}

Example 14 with RowAdapter

use of org.pentaho.di.trans.step.RowAdapter in project pentaho-kettle by pentaho.

the class CsvInputDialog method updatePreview.

/**
 * Load metadata from step window
 */
protected void updatePreview() {
    if (initializing) {
        return;
    }
    if (previewBusy.get()) {
        return;
    }
    try {
        previewBusy.set(true);
        CsvInputMeta meta = new CsvInputMeta();
        getInfo(meta);
        // 
        if (Utils.isEmpty(meta.getFilename())) {
            return;
        }
        if (Utils.isEmpty(meta.getInputFields())) {
            return;
        }
        String stepname = wStepname.getText();
        // StepMeta stepMeta = new StepMeta(stepname, meta);
        StringBuffer buffer = new StringBuffer();
        final List<Object[]> rowsData = new ArrayList<Object[]>();
        final RowMetaInterface rowMeta = new RowMeta();
        try {
            meta.getFields(rowMeta, stepname, null, null, transMeta, repository, metaStore);
            TransMeta previewTransMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, meta, stepname);
            final Trans trans = new Trans(previewTransMeta);
            trans.prepareExecution(null);
            StepInterface step = trans.getRunThread(stepname, 0);
            step.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    rowsData.add(row);
                    // 
                    if (rowsData.size() > PropsUI.getInstance().getDefaultPreviewSize()) {
                        trans.stopAll();
                    }
                }
            });
            trans.startThreads();
            trans.waitUntilFinished();
            if (trans.getErrors() > 0) {
                StringBuffer log = KettleLogStore.getAppender().getBuffer(trans.getLogChannelId(), false);
                buffer.append(log);
            }
            KettleLogStore.discardLines(trans.getLogChannelId(), false);
            LoggingRegistry.getInstance().removeIncludingChildren(trans.getLogChannelId());
        } catch (Exception e) {
            buffer.append(Const.getStackTracker(e));
        }
        TransGraph transGraph = Spoon.getInstance().getActiveTransGraph();
        if (transGraph != null) {
            if (!transGraph.isExecutionResultsPaneVisible()) {
                transGraph.showExecutionResults();
            }
            transGraph.extraViewTabFolder.setSelection(5);
            transGraph.transPreviewDelegate.addPreviewData(stepMeta, rowMeta, rowsData, buffer);
            transGraph.transPreviewDelegate.setSelectedStep(stepMeta);
            transGraph.transPreviewDelegate.refreshView();
        }
    } finally {
        previewBusy.set(false);
    }
}

Example 15 with RowAdapter

use of org.pentaho.di.trans.step.RowAdapter in project pentaho-kettle by pentaho.

the class MetaInject method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MetaInjectMeta) smi;
    data = (MetaInjectData) sdi;
    // Read the data from all input steps and keep it in memory...
    // Skip the step from which we stream data. Keep that available for runtime action.
    // 
    data.rowMap = new HashMap<String, List<RowMetaAndData>>();
    for (String prevStepName : getTransMeta().getPrevStepNames(getStepMeta())) {
        // 
        if (!data.streaming || !prevStepName.equalsIgnoreCase(data.streamingSourceStepname)) {
            List<RowMetaAndData> list = new ArrayList<RowMetaAndData>();
            RowSet rowSet = findInputRowSet(prevStepName);
            Object[] row = getRowFrom(rowSet);
            while (row != null) {
                RowMetaAndData rd = new RowMetaAndData();
                rd.setRowMeta(rowSet.getRowMeta());
                rd.setData(row);
                list.add(rd);
                row = getRowFrom(rowSet);
            }
            if (!list.isEmpty()) {
                data.rowMap.put(prevStepName, list);
            }
        }
    }
    List<StepMeta> steps = data.transMeta.getSteps();
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjection(en.getKey(), en.getValue());
    }
    /*
     * constants injection should be executed after steps, because if constant should be inserted into target with array
     * in path, constants should be inserted into all arrays items
     */
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjectionConstants(en.getKey(), en.getValue());
    }
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        en.getValue().searchInfoAndTargetSteps(steps);
    }
    for (String targetStepName : data.stepInjectionMap.keySet()) {
        if (!data.stepInjectionMetasMap.containsKey(targetStepName)) {
            oldInjection(targetStepName);
            StepMeta targetStep = StepMeta.findStep(steps, targetStepName);
            if (targetStep != null) {
                targetStep.getStepMetaInterface().searchInfoAndTargetSteps(steps);
            }
        }
    }
    if (!meta.isNoExecution()) {
        // Now we can execute this modified transformation metadata.
        // 
        final Trans injectTrans = createInjectTrans();
        injectTrans.setMetaStore(getMetaStore());
        if (getTrans().getParentJob() != null) {
            // See PDI-13224
            injectTrans.setParentJob(getTrans().getParentJob());
        }
        getTrans().addTransStoppedListener(new TransStoppedListener() {

            public void transStopped(Trans parentTrans) {
                injectTrans.stopAll();
            }
        });
        injectTrans.prepareExecution(null);
        // See if we need to stream some data over...
        // 
        RowProducer rowProducer = null;
        if (data.streaming) {
            rowProducer = injectTrans.addRowProducer(data.streamingTargetStepname, 0);
        }
        // Finally, add the mapping transformation to the active sub-transformations
        // map in the parent transformation
        // 
        getTrans().addActiveSubTransformation(getStepname(), injectTrans);
        if (!Utils.isEmpty(meta.getSourceStepName())) {
            StepInterface stepInterface = injectTrans.getStepInterface(meta.getSourceStepName(), 0);
            if (stepInterface == null) {
                throw new KettleException("Unable to find step '" + meta.getSourceStepName() + "' to read from.");
            }
            stepInterface.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    // Just pass along the data as output of this step...
                    // 
                    MetaInject.this.putRow(rowMeta, row);
                }
            });
        }
        injectTrans.startThreads();
        if (data.streaming) {
            // Deplete all the rows from the parent transformation into the modified transformation
            // 
            RowSet rowSet = findInputRowSet(data.streamingSourceStepname);
            if (rowSet == null) {
                throw new KettleException("Unable to find step '" + data.streamingSourceStepname + "' to stream data from");
            }
            Object[] row = getRowFrom(rowSet);
            while (row != null && !isStopped()) {
                rowProducer.putRow(rowSet.getRowMeta(), row);
                row = getRowFrom(rowSet);
            }
            rowProducer.finished();
        }
        // 
        while (!injectTrans.isFinished() && !injectTrans.isStopped() && !isStopped()) {
            copyResult(injectTrans);
            // Wait a little bit.
            try {
                Thread.sleep(50);
            } catch (Exception e) {
            // Ignore errors
            }
        }
        copyResult(injectTrans);
        waitUntilFinished(injectTrans);
    }
    // let the transformation complete it's execution to allow for any customizations to MDI to happen in the init methods of steps
    if (log.isDetailed()) {
        logDetailed("XML of transformation after injection: " + data.transMeta.getXML());
    }
    String targetFile = environmentSubstitute(meta.getTargetFile());
    if (!Utils.isEmpty(targetFile)) {
        writeInjectedKtr(targetFile);
    }
    // All done!
    setOutputDone();
    return false;
}