Examples with TransDebugMeta org.pentaho.di.trans.debug.TransDebugMeta used on opensource projects

Example 1 with TransDebugMeta

use of org.pentaho.di.trans.debug.TransDebugMeta in project pentaho-kettle by pentaho.

the class AbstractPreviewRowsXulDialog method previewStep.

/**
 * TODO: This method needs to communicate and control a UI wait status indicator (aka, progress monitor)
 *
 * @param data
 * @param columns
 */
protected void previewStep(List<Object[]> data, List<String> columns) {
    TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(null, (StepMetaInterface) meta, "data_sync");
    final Trans trans = new Trans(previewMeta);
    try {
        trans.prepareExecution(null);
        TransDebugMeta transDebugMeta = new TransDebugMeta(previewMeta);
        StepMeta stepMeta = previewMeta.findStep("data_sync");
        StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
        stepDebugMeta.setReadingFirstRows(true);
        stepDebugMeta.setRowCount(maxRows);
        transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
        transDebugMeta.addRowListenersToTransformation(trans);
        transDebugMeta.addBreakPointListers(new BreakPointListener() {

            public void breakPointHit(TransDebugMeta transDebugMeta, StepDebugMeta stepDebugMeta, RowMetaInterface rowBufferMeta, List<Object[]> rowBuffer) {
                System.out.println("break point hit...".concat(String.valueOf(stepDebugMeta.getRowCount())));
                trans.stopAll();
            }
        });
        trans.startThreads();
        /*
       * if (previewMeta.getTransformationType() == TransformationType.Normal) { trans.waitUntilFinished(); }
       */
        int previousPct = 0;
        while (!trans.isFinished()) {
            // How many rows are done?
            int nrDone = 0;
            int nrTotal = 0;
            for (StepDebugMeta debug : transDebugMeta.getStepDebugMetaMap().values()) {
                nrDone += debug.getRowBuffer().size();
                nrTotal += debug.getRowCount();
            }
            int pct = 100 * nrDone / nrTotal;
            int worked = pct - previousPct;
            if (worked > 0) {
                this.progressMeter.setValue(worked);
            }
            previousPct = pct;
            // Change the percentage...
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            // Ignore sleep interruption exception
            }
        }
        trans.stopAll();
        data.addAll(stepDebugMeta.getRowBuffer());
        RowMetaInterface rowMeta = stepDebugMeta.getRowBufferMeta();
        for (int i = 0; i < rowMeta.size(); i++) {
            ValueMetaInterface v = rowMeta.getValueMeta(i);
            columns.add(v.getName());
        }
    } catch (KettleException e) {
        this.logError("Data preview failed.", e);
    }
}

Example 2 with TransDebugMeta

use of org.pentaho.di.trans.debug.TransDebugMeta in project pentaho-kettle by pentaho.

the class TransGraph method debug.

public synchronized void debug(TransExecutionConfiguration executionConfiguration, TransDebugMeta transDebugMeta) {
    if (!running) {
        try {
            this.lastTransDebugMeta = transDebugMeta;
            log.setLogLevel(executionConfiguration.getLogLevel());
            if (log.isDetailed()) {
                log.logDetailed(BaseMessages.getString(PKG, "TransLog.Log.DoPreview"));
            }
            String[] args = null;
            Map<String, String> arguments = executionConfiguration.getArguments();
            if (arguments != null) {
                args = convertArguments(arguments);
            }
            transMeta.injectVariables(executionConfiguration.getVariables());
            // Set the named parameters
            Map<String, String> paramMap = executionConfiguration.getParams();
            Set<String> keys = paramMap.keySet();
            for (String key : keys) {
                transMeta.setParameterValue(key, Const.NVL(paramMap.get(key), ""));
            }
            transMeta.activateParameters();
            // 
            if (executionConfiguration.isClearingLog()) {
                transLogDelegate.clearLog();
            }
            // 
            if (trans != null) {
                KettleLogStore.discardLines(trans.getLogChannelId(), false);
                LoggingRegistry.getInstance().removeIncludingChildren(trans.getLogChannelId());
            }
            // Create a new transformation to execution
            // 
            trans = new Trans(transMeta);
            trans.setSafeModeEnabled(executionConfiguration.isSafeModeEnabled());
            trans.setPreview(true);
            trans.setGatheringMetrics(executionConfiguration.isGatheringMetrics());
            trans.setMetaStore(spoon.getMetaStore());
            trans.prepareExecution(args);
            trans.setRepository(spoon.rep);
            List<SpoonUiExtenderPluginInterface> relevantExtenders = SpoonUiExtenderPluginType.getInstance().getRelevantExtenders(TransDebugMetaWrapper.class, PREVIEW_TRANS);
            TransDebugMetaWrapper transDebugMetaWrapper = new TransDebugMetaWrapper(trans, transDebugMeta);
            for (SpoonUiExtenderPluginInterface relevantExtender : relevantExtenders) {
                relevantExtender.uiEvent(transDebugMetaWrapper, PREVIEW_TRANS);
            }
            // Add the row listeners to the allocated threads
            // 
            transDebugMeta.addRowListenersToTransformation(trans);
            // What method should we call back when a break-point is hit?
            transDebugMeta.addBreakPointListers(new BreakPointListener() {

                @Override
                public void breakPointHit(TransDebugMeta transDebugMeta, StepDebugMeta stepDebugMeta, RowMetaInterface rowBufferMeta, List<Object[]> rowBuffer) {
                    showPreview(transDebugMeta, stepDebugMeta, rowBufferMeta, rowBuffer);
                }
            });
            // 
            if (transPreviewDelegate.isActive()) {
                transPreviewDelegate.capturePreviewData(trans, transMeta.getSteps());
            }
            // Start the threads for the steps...
            // 
            startThreads();
            debug = true;
            // Show the execution results view...
            // 
            shell.getDisplay().asyncExec(new Runnable() {

                @Override
                public void run() {
                    addAllTabs();
                }
            });
        } catch (Exception e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "TransLog.Dialog.UnexpectedErrorDuringPreview.Title"), BaseMessages.getString(PKG, "TransLog.Dialog.UnexpectedErrorDuringPreview.Message"), e);
        }
    } else {
        MessageBox m = new MessageBox(shell, SWT.OK | SWT.ICON_WARNING);
        m.setText(BaseMessages.getString(PKG, "TransLog.Dialog.DoNoPreviewWhileRunning.Title"));
        m.setMessage(BaseMessages.getString(PKG, "TransLog.Dialog.DoNoPreviewWhileRunning.Message"));
        m.open();
    }
    checkErrorVisuals();
}

Example 3 with TransDebugMeta

use of org.pentaho.di.trans.debug.TransDebugMeta in project pentaho-kettle by pentaho.

the class TransPreviewProgressDialog method doPreview.

private void doPreview(final IProgressMonitor progressMonitor, final boolean showErrorDialogs) {
    progressMonitor.beginTask(BaseMessages.getString(PKG, "TransPreviewProgressDialog.Monitor.BeginTask.Title"), 100);
    // This transformation is ready to run in preview!
    trans = new Trans(transMeta);
    trans.setPreview(true);
    // 
    try {
        trans.prepareExecution(null);
    } catch (final KettleException e) {
        if (showErrorDialogs) {
            shell.getDisplay().asyncExec(new Runnable() {

                public void run() {
                    new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "TransPreviewProgressDialog.Exception.ErrorPreparingTransformation"), e);
                }
            });
        }
        // 
        return;
    }
    // Add the preview / debugging information...
    // 
    transDebugMeta = new TransDebugMeta(transMeta);
    for (int i = 0; i < previewStepNames.length; i++) {
        StepMeta stepMeta = transMeta.findStep(previewStepNames[i]);
        StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
        stepDebugMeta.setReadingFirstRows(true);
        stepDebugMeta.setRowCount(previewSize[i]);
        transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
    }
    // set the appropriate listeners on the transformation...
    // 
    transDebugMeta.addRowListenersToTransformation(trans);
    // 
    try {
        trans.startThreads();
    } catch (final KettleException e) {
        shell.getDisplay().asyncExec(new Runnable() {

            public void run() {
                new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "TransPreviewProgressDialog.Exception.ErrorPreparingTransformation"), e);
            }
        });
        // 
        return;
    }
    int previousPct = 0;
    final List<String> previewComplete = new ArrayList<String>();
    while (previewComplete.size() < previewStepNames.length && !trans.isFinished() && !progressMonitor.isCanceled()) {
        // We add a break-point that is called every time we have a step with a full preview row buffer
        // That makes it easy and fast to see if we have all the rows we need
        // 
        transDebugMeta.addBreakPointListers(new BreakPointListener() {

            public void breakPointHit(TransDebugMeta transDebugMeta, StepDebugMeta stepDebugMeta, RowMetaInterface rowBufferMeta, List<Object[]> rowBuffer) {
                String stepName = stepDebugMeta.getStepMeta().getName();
                previewComplete.add(stepName);
                progressMonitor.subTask(BaseMessages.getString(PKG, "TransPreviewProgressDialog.SubTask.StepPreviewFinished", stepName));
            }
        });
        // How many rows are done?
        int nrDone = 0;
        int nrTotal = 0;
        for (StepDebugMeta stepDebugMeta : transDebugMeta.getStepDebugMetaMap().values()) {
            nrDone += stepDebugMeta.getRowBuffer().size();
            nrTotal += stepDebugMeta.getRowCount();
        }
        int pct = 100 * nrDone / nrTotal;
        int worked = pct - previousPct;
        if (worked > 0) {
            progressMonitor.worked(worked);
        }
        previousPct = pct;
        // Change the percentage...
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
        // Ignore errors
        }
        if (progressMonitor.isCanceled()) {
            cancelled = true;
            trans.stopAll();
        }
    }
    trans.stopAll();
    // Capture preview activity to a String:
    loggingText = KettleLogStore.getAppender().getBuffer(trans.getLogChannel().getLogChannelId(), true).toString();
    progressMonitor.done();
}

Example 4 with TransDebugMeta

use of org.pentaho.di.trans.debug.TransDebugMeta in project pentaho-kettle by pentaho.

the class SpoonTransformationDelegate method executeTransformation.

public void executeTransformation(final TransMeta transMeta, final boolean local, final boolean remote, final boolean cluster, final boolean preview, final boolean debug, final Date replayDate, final boolean safe, LogLevel logLevel) throws KettleException {
    if (transMeta == null) {
        return;
    }
    // See if we need to ask for debugging information...
    // 
    TransDebugMeta transDebugMeta = null;
    TransExecutionConfiguration executionConfiguration = null;
    if (preview) {
        executionConfiguration = spoon.getTransPreviewExecutionConfiguration();
    } else if (debug) {
        executionConfiguration = spoon.getTransDebugExecutionConfiguration();
    } else {
        executionConfiguration = spoon.getTransExecutionConfiguration();
    }
    // Set defaults so the run configuration can set it up correctly
    executionConfiguration.setExecutingLocally(true);
    executionConfiguration.setExecutingRemotely(false);
    executionConfiguration.setExecutingClustered(false);
    // Set repository and safe mode information in both the exec config and the metadata
    transMeta.setRepository(spoon.rep);
    transMeta.setMetaStore(spoon.metaStore);
    executionConfiguration.setRepository(spoon.rep);
    executionConfiguration.setSafeModeEnabled(safe);
    if (debug) {
        // See if we have debugging information stored somewhere?
        // 
        transDebugMeta = transDebugMetaMap.get(transMeta);
        if (transDebugMeta == null) {
            transDebugMeta = new TransDebugMeta(transMeta);
            transDebugMetaMap.put(transMeta, transDebugMeta);
        }
        // Set the default number of rows to retrieve on all selected steps...
        // 
        List<StepMeta> selectedSteps = transMeta.getSelectedSteps();
        if (selectedSteps != null && selectedSteps.size() > 0) {
            transDebugMeta.getStepDebugMetaMap().clear();
            for (StepMeta stepMeta : transMeta.getSelectedSteps()) {
                StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
                stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize());
                stepDebugMeta.setPausingOnBreakPoint(true);
                stepDebugMeta.setReadingFirstRows(false);
                transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
            }
        }
    } else if (preview) {
        // See if we have preview information stored somewhere?
        // 
        transDebugMeta = transPreviewMetaMap.get(transMeta);
        if (transDebugMeta == null) {
            transDebugMeta = new TransDebugMeta(transMeta);
            transPreviewMetaMap.put(transMeta, transDebugMeta);
        }
        // Set the default number of preview rows on all selected steps...
        // 
        List<StepMeta> selectedSteps = transMeta.getSelectedSteps();
        if (selectedSteps != null && selectedSteps.size() > 0) {
            transDebugMeta.getStepDebugMetaMap().clear();
            for (StepMeta stepMeta : transMeta.getSelectedSteps()) {
                StepDebugMeta stepDebugMeta = new StepDebugMeta(stepMeta);
                stepDebugMeta.setRowCount(PropsUI.getInstance().getDefaultPreviewSize());
                stepDebugMeta.setPausingOnBreakPoint(false);
                stepDebugMeta.setReadingFirstRows(true);
                transDebugMeta.getStepDebugMetaMap().put(stepMeta, stepDebugMeta);
            }
        }
    }
    int debugAnswer = TransDebugDialog.DEBUG_CONFIG;
    if (debug || preview) {
        // pass repository for mappings
        transDebugMeta.getTransMeta().setRepository(spoon.rep);
        TransDebugDialog transDebugDialog = new TransDebugDialog(spoon.getShell(), transDebugMeta);
        debugAnswer = transDebugDialog.open();
        if (debugAnswer != TransDebugDialog.DEBUG_CANCEL) {
            executionConfiguration.setExecutingLocally(true);
            executionConfiguration.setExecutingRemotely(false);
            executionConfiguration.setExecutingClustered(false);
        } else {
            // 
            return;
        }
    }
    Object[] data = spoon.variables.getData();
    String[] fields = spoon.variables.getRowMeta().getFieldNames();
    Map<String, String> variableMap = new HashMap<String, String>();
    // the default
    variableMap.putAll(executionConfiguration.getVariables());
    for (int idx = 0; idx < fields.length; idx++) {
        String value = executionConfiguration.getVariables().get(fields[idx]);
        if (Utils.isEmpty(value)) {
            value = data[idx].toString();
        }
        variableMap.put(fields[idx], value);
    }
    executionConfiguration.setVariables(variableMap);
    executionConfiguration.getUsedVariables(transMeta);
    executionConfiguration.getUsedArguments(transMeta, spoon.getArguments());
    executionConfiguration.setReplayDate(replayDate);
    executionConfiguration.setLogLevel(logLevel);
    boolean execConfigAnswer = true;
    if (debugAnswer == TransDebugDialog.DEBUG_CONFIG && replayDate == null && transMeta.isShowDialog()) {
        TransExecutionConfigurationDialog dialog = new TransExecutionConfigurationDialog(spoon.getShell(), executionConfiguration, transMeta);
        execConfigAnswer = dialog.open();
    }
    if (execConfigAnswer) {
        TransGraph activeTransGraph = spoon.getActiveTransGraph();
        activeTransGraph.transLogDelegate.addTransLog();
        // Set the named parameters
        Map<String, String> paramMap = executionConfiguration.getParams();
        for (String key : paramMap.keySet()) {
            transMeta.setParameterValue(key, Const.NVL(paramMap.get(key), ""));
        }
        transMeta.activateParameters();
        // 
        if (executionConfiguration.getLogLevel() != null) {
            transMeta.setLogLevel(executionConfiguration.getLogLevel());
        }
        // Set the run options
        transMeta.setClearingLog(executionConfiguration.isClearingLog());
        transMeta.setSafeModeEnabled(executionConfiguration.isSafeModeEnabled());
        transMeta.setGatheringMetrics(executionConfiguration.isGatheringMetrics());
        ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.SpoonTransMetaExecutionStart.id, transMeta);
        ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.SpoonTransExecutionConfiguration.id, executionConfiguration);
        try {
            ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.SpoonTransBeforeStart.id, new Object[] { executionConfiguration, transMeta, transMeta, spoon.getRepository() });
        } catch (KettleException e) {
            log.logError(e.getMessage(), transMeta.getFilename());
            return;
        }
        if (!executionConfiguration.isExecutingLocally() && !executionConfiguration.isExecutingRemotely()) {
            if (transMeta.hasChanged()) {
                activeTransGraph.showSaveFileMessage();
            }
        }
        // 
        if (debug || preview) {
            if (transDebugMeta.getNrOfUsedSteps() == 0) {
                MessageBox box = new MessageBox(spoon.getShell(), SWT.ICON_WARNING | SWT.YES | SWT.NO);
                box.setText(BaseMessages.getString(PKG, "Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Title"));
                box.setMessage(BaseMessages.getString(PKG, "Spoon.Dialog.Warning.NoPreviewOrDebugSteps.Message"));
                int answer = box.open();
                if (answer != SWT.YES) {
                    return;
                }
            }
        }
        // 
        if (executionConfiguration.isExecutingLocally()) {
            if (debug || preview) {
                activeTransGraph.debug(executionConfiguration, transDebugMeta);
            } else {
                activeTransGraph.start(executionConfiguration);
            }
        // Are we executing remotely?
        // 
        } else if (executionConfiguration.isExecutingRemotely()) {
            activeTransGraph.handleTransMetaChanges(transMeta);
            if (transMeta.hasChanged()) {
                showSaveTransformationBeforeRunningDialog(spoon.getShell());
            } else if (executionConfiguration.getRemoteServer() != null) {
                String carteObjectId = Trans.sendToSlaveServer(transMeta, executionConfiguration, spoon.rep, spoon.metaStore);
                monitorRemoteTrans(transMeta, carteObjectId, executionConfiguration.getRemoteServer());
                spoon.delegates.slaves.addSpoonSlave(executionConfiguration.getRemoteServer());
            } else {
                MessageBox mb = new MessageBox(spoon.getShell(), SWT.OK | SWT.ICON_INFORMATION);
                mb.setMessage(BaseMessages.getString(PKG, "Spoon.Dialog.NoRemoteServerSpecified.Message"));
                mb.setText(BaseMessages.getString(PKG, "Spoon.Dialog.NoRemoteServerSpecified.Title"));
                mb.open();
            }
        // Are we executing clustered?
        // 
        } else if (executionConfiguration.isExecutingClustered()) {
            activeTransGraph.handleTransMetaChanges(transMeta);
            if (transMeta.hasChanged()) {
                showSaveTransformationBeforeRunningDialog(spoon.getShell());
            } else {
                splitTrans(transMeta, executionConfiguration);
            }
        }
    }
}