Examples with PipelineMeta org.apache.hop.pipeline.PipelineMeta used on opensource projects

Example 21 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class TestingGuiPlugin method createUnitTest.

@GuiToolbarElement(root = HopGuiPipelineGraph.GUI_PLUGIN_TOOLBAR_PARENT_ID, id = ID_TOOLBAR_ITEM_UNIT_TESTS_CREATE, toolTip = "i18n::TestingGuiPlugin.ToolbarElement.UnitTest.Create.Tooltip", image = "Test_tube_icon_create.svg", separator = true)
public void createUnitTest() {
    HopGui hopGui = HopGui.getInstance();
    PipelineMeta pipelineMeta = getActivePipelineMeta();
    if (pipelineMeta == null) {
        return;
    }
    MetadataManager<PipelineUnitTest> manager = new MetadataManager<>(hopGui.getVariables(), hopGui.getMetadataProvider(), PipelineUnitTest.class);
    PipelineUnitTest test = manager.newMetadata();
    if (test != null) {
        // Activate the test
        refreshUnitTestsList();
        selectUnitTest(pipelineMeta, test);
    }
}

Example 22 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class PipelineLogEditor method createPipelineFile.

/**
 * Create a new pipeline file: ask the user for a name. Add a standard transform and a dummy to
 * show how it works.
 *
 * @param parent
 */
private void createPipelineFile(Composite parent) {
    try {
        PipelineMeta pipelineMeta = new PipelineMeta();
        // Add a Pipeline Logging transform...
        // 
        PipelineLoggingMeta pipelineLoggingMeta = new PipelineLoggingMeta();
        pipelineLoggingMeta.setLoggingTransforms(true);
        TransformMeta pipelineLogging = new TransformMeta("Pipeline Logging", pipelineLoggingMeta);
        pipelineLogging.setLocation(200, 150);
        pipelineMeta.addTransform(pipelineLogging);
        // Add a dummy
        // 
        DummyMeta dummyMeta = new DummyMeta();
        TransformMeta dummy = new TransformMeta("Save logging here", dummyMeta);
        dummy.setLocation(500, 150);
        pipelineMeta.addTransform(dummy);
        // Add a hop between both transforms...
        // 
        pipelineMeta.addPipelineHop(new PipelineHopMeta(pipelineLogging, dummy));
        // Save it...
        // 
        HopPipelineFileType<PipelineMeta> type = new HopPipelineFileType<>();
        String filename = BaseDialog.presentFileDialog(// save
        true, parent.getShell(), wFilename, manager.getVariables(), type.getFilterExtensions(), type.getFilterNames(), true);
        if (filename != null) {
            // User specified a pipeline filename
            // 
            String realFilename = manager.getVariables().resolve(filename);
            pipelineMeta.setFilename(realFilename);
            pipelineMeta.clearChanged();
            HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
            // Switch to the perspective
            // 
            perspective.activate();
            // Open it in the Hop GUI
            // 
            HopGui.getDataOrchestrationPerspective().addPipeline(hopGui, pipelineMeta, type);
            // Save the file
            hopGui.fileDelegate.fileSave();
        }
    } catch (Exception e) {
        new ErrorDialog(parent.getShell(), "Error", "Error creating pipeline", e);
    }
}

Example 23 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class DataProbeGuiPlugin method addDataProbeForTransform.

@GuiContextAction(id = "pipeline-graph-transform-9000-add-probe", parentId = HopGuiPipelineTransformContext.CONTEXT_ID, type = GuiActionType.Info, name = "Add data probe", tooltip = "Streams the rows to a pipeline defined in a selected pipeline probe", image = "probe.svg", category = "Preview", categoryOrder = "3")
public void addDataProbeForTransform(HopGuiPipelineTransformContext context) {
    PipelineMeta pipelineMeta = context.getPipelineMeta();
    TransformMeta transformMeta = context.getTransformMeta();
    HopGui hopGui = HopGui.getInstance();
    try {
        // Present the user with a list of pipeline probes...
        // 
        IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
        IHopMetadataSerializer<PipelineProbe> serializer = metadataProvider.getSerializer(PipelineProbe.class);
        MetadataManager<PipelineProbe> manager = new MetadataManager<>(hopGui.getVariables(), metadataProvider, PipelineProbe.class);
        PipelineProbe pipelineProbe = null;
        List<String> pipelineProbeNames = serializer.listObjectNames();
        if (pipelineProbeNames.isEmpty()) {
            MessageBox box = new MessageBox(hopGui.getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
            box.setText("No probes available");
            box.setMessage("There are no pipeline probe objects defined yet.  Do you want to create one?");
            int answer = box.open();
            if ((answer & SWT.YES) != 0) {
                // Create a new pipeline probe...
                // 
                pipelineProbe = new PipelineProbe();
                addLocation(hopGui.getVariables(), pipelineProbe, pipelineMeta, transformMeta);
                manager.newMetadata(pipelineProbe);
                return;
            } else {
                return;
            }
        } else {
            EnterSelectionDialog dialog = new EnterSelectionDialog(hopGui.getShell(), pipelineProbeNames.toArray(new String[0]), "Select pipeline probe", "Select the pipeline probe to add this pipeline transform to");
            String pipelineProbeName = dialog.open();
            if (pipelineProbeName != null) {
                pipelineProbe = serializer.load(pipelineProbeName);
            }
        }
        if (pipelineProbe != null) {
            // See if it's open in the metadata perspective...
            // 
            MetadataPerspective perspective = (MetadataPerspective) hopGui.getPerspectiveManager().findPerspective(MetadataPerspective.class);
            String key = PipelineProbe.class.getAnnotation(HopMetadata.class).key();
            PipelineProbeEditor editor = (PipelineProbeEditor) perspective.findEditor(key, pipelineProbe.getName());
            if (editor != null) {
                // We're going to change the current metadata and flag it as changed...
                // 
                pipelineProbe = new PipelineProbe();
                editor.getWidgetsContent(pipelineProbe);
                // Add the location
                // 
                addLocation(hopGui.getVariables(), pipelineProbe, pipelineMeta, transformMeta);
                // Replace and refresh the dialog
                // 
                editor.setMetadata(pipelineProbe);
                editor.setWidgetsContent();
                // Set changed...
                // 
                editor.setChanged();
                // Switch to the editor...
                // 
                perspective.activate();
                perspective.setActiveEditor(editor);
                return;
            } else {
                // Not opened in the perspective, simply add the data probe location...
                // 
                addLocation(hopGui.getVariables(), pipelineProbe, pipelineMeta, transformMeta);
                // ... and save the pipeline probe
                // 
                serializer.save(pipelineProbe);
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), "Error", "Error adding pipeline probe to transform '" + transformMeta.getName() + "'", e);
    }
}

Example 24 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class PipelineProbeEditor method createPipelineFile.

/**
 * Create a new pipeline file: ask the user for a name. Add a standard transform and a dummy to
 * show how it works.
 *
 * @param parent
 */
private void createPipelineFile(Composite parent) {
    try {
        PipelineMeta pipelineMeta = new PipelineMeta();
        // Add a Pipeline Data Probe transform...
        // 
        PipelineDataProbeMeta pipelineDataProbeMeta = new PipelineDataProbeMeta();
        pipelineDataProbeMeta.setLoggingTransforms(true);
        TransformMeta pipelineLogging = new TransformMeta("Pipeline Data Probe", pipelineDataProbeMeta);
        pipelineLogging.setLocation(200, 150);
        pipelineMeta.addTransform(pipelineLogging);
        // Add a dummy
        // 
        DummyMeta dummyMeta = new DummyMeta();
        TransformMeta dummy = new TransformMeta("Process values here", dummyMeta);
        dummy.setLocation(500, 150);
        pipelineMeta.addTransform(dummy);
        // Add a hop between both transforms...
        // 
        pipelineMeta.addPipelineHop(new PipelineHopMeta(pipelineLogging, dummy));
        // Save it...
        // 
        HopPipelineFileType<PipelineMeta> type = new HopPipelineFileType<>();
        String filename = BaseDialog.presentFileDialog(// save
        true, parent.getShell(), wFilename, manager.getVariables(), type.getFilterExtensions(), type.getFilterNames(), true);
        if (filename != null) {
            // User specified a pipeline filename
            // 
            String realFilename = manager.getVariables().resolve(filename);
            pipelineMeta.setFilename(realFilename);
            pipelineMeta.clearChanged();
            HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
            // Switch to the perspective
            // 
            perspective.activate();
            // Open it in the Hop GUI
            // 
            HopGui.getDataOrchestrationPerspective().addPipeline(hopGui, pipelineMeta, type);
            // Save the file
            hopGui.fileDelegate.fileSave();
        }
    } catch (Exception e) {
        new ErrorDialog(parent.getShell(), "Error", "Error creating pipeline", e);
    }
}

Example 25 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class PipelineDataProbeXp method executeProbingPipeline.

/**
 * Execute a probing pipeline for the current pipeline. Add a listener to the transform copies.
 * Send the data to the PipelineDataProbe transform(s) in the probing pipeline
 *
 * @param pipelineProbe
 * @param dataProbeLocation
 * @param loggingPipelineFilename The pipeline to start for the location
 * @param pipeline The parent pipeline to listen to
 * @param variables
 * @throws HopException
 */
private synchronized void executeProbingPipeline(PipelineProbe pipelineProbe, DataProbeLocation dataProbeLocation, String loggingPipelineFilename, IPipelineEngine<PipelineMeta> pipeline, IVariables variables) throws HopException {
    PipelineMeta probingPipelineMeta = new PipelineMeta(loggingPipelineFilename, pipeline.getMetadataProvider(), true, variables);
    // Create a local pipeline engine...
    // 
    LocalPipelineEngine probingPipeline = new LocalPipelineEngine(probingPipelineMeta, variables, pipeline);
    // Flag it as a probing and logging pipeline so we don't try to probe or log ourselves...
    // 
    probingPipeline.getExtensionDataMap().put(PIPELINE_DATA_PROBE_FLAG, "Y");
    probingPipeline.getExtensionDataMap().put(PipelineStartLoggingXp.PIPELINE_LOGGING_FLAG, "Y");
    // Only log errors
    // 
    probingPipeline.setLogLevel(LogLevel.ERROR);
    probingPipeline.prepareExecution();
    List<IEngineComponent> componentCopies = pipeline.getComponentCopies(dataProbeLocation.getSourceTransformName());
    for (IEngineComponent componentCopy : componentCopies) {
        // 
        for (TransformMetaDataCombi combi : probingPipeline.getTransforms()) {
            if (combi.transform instanceof PipelineDataProbe) {
                // Give the transform a bit more information to work with...
                // 
                PipelineDataProbe pipelineDataProbe = (PipelineDataProbe) combi.transform;
                pipelineDataProbe.setSourcePipelineName(pipeline.getPipelineMeta().getName());
                pipelineDataProbe.setSourceTransformLogChannelId(pipeline.getLogChannelId());
                pipelineDataProbe.setSourceTransformName(componentCopy.getName());
                pipelineDataProbe.setSourceTransformCopy(componentCopy.getCopyNr());
                try {
                    final RowProducer rowProducer = probingPipeline.addRowProducer(combi.transformName, combi.copy);
                    // For every copy of the component, add an input row set to the parent pipeline...
                    // 
                    componentCopy.addRowListener(new RowAdapter() {

                        @Override
                        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            // Pass this row to the row producer...
                            // 
                            rowProducer.putRow(rowMeta, row);
                        }
                    });
                    // If the pipeline we're the transform is and we can safely stop streaming...
                    // 
                    pipeline.addExecutionFinishedListener(pe -> rowProducer.finished());
                } catch (HopException e) {
                    throw new HopTransformException("Error adding row producer to transform '" + combi.transformName + "'", e);
                }
            }
        }
    }
    // Execute the logging pipeline to save the logging information
    // 
    probingPipeline.startThreads();
    // We'll not wait around until this is finished...
    // The pipeline should stop automatically when the parent does
    // 
    pipeline.addExecutionStoppedListener(e -> probingPipeline.stopAll());
}