Examples with PipelineDebugMeta org.apache.hop.pipeline.debug.PipelineDebugMeta used on opensource projects

Example 1 with PipelineDebugMeta

use of org.apache.hop.pipeline.debug.PipelineDebugMeta in project hop by apache.

the class PipelinePreviewProgressDialog method doPreview.

private void doPreview(final HopGui hopGui, final IProgressMonitor progressMonitor, final boolean showErrorDialogs) {
    progressMonitor.beginTask(BaseMessages.getString(PKG, "PipelinePreviewProgressDialog.Monitor.BeginTask.Title"), 100);
    // This pipeline is ready to run in preview!
    // 
    pipeline = new LocalPipelineEngine(pipelineMeta, variables, hopGui.getLoggingObject());
    pipeline.setPreview(true);
    pipeline.setMetadataProvider(hopGui.getMetadataProvider());
    // 
    try {
        pipeline.prepareExecution();
    } catch (final HopException e) {
        if (showErrorDialogs) {
            shell.getDisplay().asyncExec(() -> new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "PipelinePreviewProgressDialog.Exception.ErrorPreparingPipeline"), e));
        }
        // 
        return;
    }
    // Add the preview / debugging information...
    // 
    pipelineDebugMeta = new PipelineDebugMeta(pipelineMeta);
    for (int i = 0; i < previewTransformNames.length; i++) {
        TransformMeta transformMeta = pipelineMeta.findTransform(previewTransformNames[i]);
        TransformDebugMeta transformDebugMeta = new TransformDebugMeta(transformMeta);
        transformDebugMeta.setReadingFirstRows(true);
        transformDebugMeta.setRowCount(previewSize[i]);
        pipelineDebugMeta.getTransformDebugMetaMap().put(transformMeta, transformDebugMeta);
    }
    int previousPct = 0;
    final List<String> previewComplete = new ArrayList<>();
    // We add a break-point that is called every time we have a transform with a full preview row
    // buffer
    // That makes it easy and fast to see if we have all the rows we need
    // 
    pipelineDebugMeta.addBreakPointListers((pipelineDebugMeta, transformDebugMeta, rowBufferMeta, rowBuffer) -> {
        String transformName = transformDebugMeta.getTransformMeta().getName();
        previewComplete.add(transformName);
        progressMonitor.subTask(BaseMessages.getString(PKG, "PipelinePreviewProgressDialog.SubTask.TransformPreviewFinished", transformName));
    });
    // set the appropriate listeners on the pipeline...
    // 
    pipelineDebugMeta.addRowListenersToPipeline(pipeline);
    // 
    try {
        pipeline.startThreads();
    } catch (final HopException e) {
        shell.getDisplay().asyncExec(() -> new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "PipelinePreviewProgressDialog.Exception.ErrorPreparingPipeline"), e));
        // 
        return;
    }
    while (previewComplete.size() < previewTransformNames.length && !pipeline.isFinished() && !progressMonitor.isCanceled()) {
        // How many rows are done?
        int nrDone = 0;
        int nrTotal = 0;
        for (TransformDebugMeta transformDebugMeta : pipelineDebugMeta.getTransformDebugMetaMap().values()) {
            nrDone += transformDebugMeta.getRowBuffer().size();
            nrTotal += transformDebugMeta.getRowCount();
        }
        int pct = 100 * nrDone / nrTotal;
        int worked = pct - previousPct;
        if (worked > 0) {
            progressMonitor.worked(worked);
        }
        previousPct = pct;
        // Change the percentage...
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
        // Ignore errors
        }
        if (progressMonitor.isCanceled()) {
            cancelled = true;
            pipeline.stopAll();
        }
    }
    pipeline.stopAll();
    // Capture preview activity to a String:
    loggingText = HopLogStore.getAppender().getBuffer(pipeline.getLogChannel().getLogChannelId(), true).toString();
    progressMonitor.done();
}

Example 2 with PipelineDebugMeta

use of org.apache.hop.pipeline.debug.PipelineDebugMeta in project hop by apache.

the class HopGuiPipelineRunDelegate method executePipeline.

public PipelineExecutionConfiguration executePipeline(final ILogChannel log, final PipelineMeta pipelineMeta, final boolean preview, final boolean debug, LogLevel logLevel) throws HopException {
    if (pipelineMeta == null) {
        return null;
    }
    // See if we need to ask for debugging information...
    // 
    PipelineDebugMeta pipelineDebugMeta = null;
    PipelineExecutionConfiguration executionConfiguration = null;
    if (preview) {
        executionConfiguration = getPipelinePreviewExecutionConfiguration();
    } else if (debug) {
        executionConfiguration = getPipelineDebugExecutionConfiguration();
    } else {
        executionConfiguration = getPipelineExecutionConfiguration();
    }
    // Set MetaStore and safe mode information in both the exec config and the metadata
    // 
    pipelineMeta.setMetadataProvider(hopGui.getMetadataProvider());
    if (debug) {
        // See if we have debugging information stored somewhere?
        // 
        pipelineDebugMeta = pipelineDebugMetaMap.get(pipelineMeta);
        if (pipelineDebugMeta == null) {
            pipelineDebugMeta = new PipelineDebugMeta(pipelineMeta);
            pipelineDebugMetaMap.put(pipelineMeta, pipelineDebugMeta);
        }
        // Set the default number of rows to retrieve on all selected transforms...
        // 
        List<TransformMeta> selectedTransforms = pipelineMeta.getSelectedTransforms();
        if (selectedTransforms != null && selectedTransforms.size() > 0) {
            pipelineDebugMeta.getTransformDebugMetaMap().clear();
            for (TransformMeta transformMeta : pipelineMeta.getSelectedTransforms()) {
                TransformDebugMeta transformDebugMeta = new TransformDebugMeta(transformMeta);
                transformDebugMeta.setRowCount(PropsUi.getInstance().getDefaultPreviewSize());
                transformDebugMeta.setPausingOnBreakPoint(true);
                transformDebugMeta.setReadingFirstRows(false);
                pipelineDebugMeta.getTransformDebugMetaMap().put(transformMeta, transformDebugMeta);
            }
        }
    } else if (preview) {
        // See if we have preview information stored somewhere?
        // 
        pipelineDebugMeta = pipelinePreviewMetaMap.get(pipelineMeta);
        if (pipelineDebugMeta == null) {
            pipelineDebugMeta = new PipelineDebugMeta(pipelineMeta);
            pipelinePreviewMetaMap.put(pipelineMeta, pipelineDebugMeta);
        }
        // Set the default number of preview rows on all selected transforms...
        // 
        List<TransformMeta> selectedTransforms = pipelineMeta.getSelectedTransforms();
        if (selectedTransforms != null && selectedTransforms.size() > 0) {
            pipelineDebugMeta.getTransformDebugMetaMap().clear();
            for (TransformMeta transformMeta : pipelineMeta.getSelectedTransforms()) {
                TransformDebugMeta transformDebugMeta = new TransformDebugMeta(transformMeta);
                transformDebugMeta.setRowCount(PropsUi.getInstance().getDefaultPreviewSize());
                transformDebugMeta.setPausingOnBreakPoint(false);
                transformDebugMeta.setReadingFirstRows(true);
                pipelineDebugMeta.getTransformDebugMetaMap().put(transformMeta, transformDebugMeta);
            }
        }
    }
    int debugAnswer = PipelineDebugDialog.DEBUG_CONFIG;
    if (debug || preview) {
        PipelineDebugDialog pipelineDebugDialog = new PipelineDebugDialog(hopGui.getShell(), pipelineGraph.getVariables(), pipelineDebugMeta);
        debugAnswer = pipelineDebugDialog.open();
        if (debugAnswer == PipelineDebugDialog.DEBUG_CANCEL) {
            // 
            return null;
        }
    }
    Map<String, String> variableMap = new HashMap<>();
    // the default
    variableMap.putAll(executionConfiguration.getVariablesMap());
    executionConfiguration.setVariablesMap(variableMap);
    executionConfiguration.getUsedVariables(pipelineGraph.getVariables(), pipelineMeta);
    executionConfiguration.setLogLevel(logLevel);
    boolean execConfigAnswer = true;
    if (debugAnswer == PipelineDebugDialog.DEBUG_CONFIG && pipelineMeta.isShowDialog()) {
        PipelineExecutionConfigurationDialog dialog = new PipelineExecutionConfigurationDialog(hopGui.getShell(), executionConfiguration, pipelineMeta);
        execConfigAnswer = dialog.open();
    }
    if (execConfigAnswer) {
        pipelineGraph.pipelineGridDelegate.addPipelineGrid();
        pipelineGraph.pipelineLogDelegate.addPipelineLog();
        pipelineGraph.extraViewTabFolder.setSelection(0);
        // Set the run options
        pipelineMeta.setClearingLog(executionConfiguration.isClearingLog());
        ExtensionPointHandler.callExtensionPoint(log, pipelineGraph.getVariables(), HopExtensionPoint.HopGuiPipelineMetaExecutionStart.id, pipelineMeta);
        ExtensionPointHandler.callExtensionPoint(log, pipelineGraph.getVariables(), HopExtensionPoint.HopGuiPipelineExecutionConfiguration.id, executionConfiguration);
        // distinction
        if (debug || preview) {
            if (pipelineDebugMeta.getNrOfUsedTransforms() == 0) {
                MessageBox box = new MessageBox(hopGui.getShell(), SWT.ICON_WARNING | SWT.YES | SWT.NO);
                box.setText(BaseMessages.getString(PKG, "HopGui.Dialog.Warning.NoPreviewOrDebugTransforms.Title"));
                box.setMessage(BaseMessages.getString(PKG, "HopGui.Dialog.Warning.NoPreviewOrDebugTransforms.Message"));
                int answer = box.open();
                if (answer != SWT.YES) {
                    return null;
                }
            }
            pipelineGraph.debug(executionConfiguration, pipelineDebugMeta);
        } else {
            pipelineGraph.start(executionConfiguration);
        }
    }
    return executionConfiguration;
}