Examples with PipelineRunConfiguration org.apache.hop.pipeline.config.PipelineRunConfiguration used on opensource projects

Example 1 with PipelineRunConfiguration

use of org.apache.hop.pipeline.config.PipelineRunConfiguration in project hop by apache.

the class ProjectsGuiPlugin method addNewProject.

@GuiToolbarElement(root = HopGui.ID_MAIN_TOOLBAR, id = ID_TOOLBAR_PROJECT_ADD, toolTip = "i18n::HopGui.Toolbar.Project.Add.Tooltip", image = "project-add.svg")
public void addNewProject() {
    HopGui hopGui = HopGui.getInstance();
    IVariables variables = hopGui.getVariables();
    try {
        ProjectsConfig config = ProjectsConfigSingleton.getConfig();
        String standardProjectsFolder = variables.resolve(config.getStandardProjectsFolder());
        String defaultProjectConfigFilename = variables.resolve(config.getDefaultProjectConfigFile());
        ProjectConfig projectConfig = new ProjectConfig("", standardProjectsFolder, defaultProjectConfigFilename);
        Project project = new Project();
        project.setParentProjectName(config.getStandardParentProject());
        ProjectDialog projectDialog = new ProjectDialog(hopGui.getShell(), project, projectConfig, variables, false);
        String projectName = projectDialog.open();
        if (projectName != null) {
            config.addProjectConfig(projectConfig);
            HopConfig.getInstance().saveToFile();
            // Save the project-config.json file as well in the project itself
            // 
            FileObject configFile = HopVfs.getFileObject(projectConfig.getActualProjectConfigFilename(variables));
            if (!configFile.exists()) {
                // Create the empty configuration file if it does not exists
                project.saveToFile();
            } else {
                // If projects exists load configuration
                MessageBox box = new MessageBox(hopGui.getShell(), SWT.ICON_QUESTION | SWT.OK);
                box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.ProjectExists.Dialog.Header"));
                box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.ProjectExists.Dialog.Message"));
                box.open();
                project.readFromFile();
            }
            refreshProjectsList();
            selectProjectInList(projectName);
            enableHopGuiProject(projectName, project, null);
            // Now see if these project contains any local run configurations.
            // If not we can add those automatically
            // 
            // First pipeline
            // 
            IHopMetadataSerializer<PipelineRunConfiguration> prcSerializer = hopGui.getMetadataProvider().getSerializer(PipelineRunConfiguration.class);
            List<PipelineRunConfiguration> pipelineRunConfigs = prcSerializer.loadAll();
            boolean localFound = false;
            for (PipelineRunConfiguration pipelineRunConfig : pipelineRunConfigs) {
                if (pipelineRunConfig.getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration) {
                    localFound = true;
                }
            }
            if (!localFound) {
                PipelineExecutionConfigurationDialog.createLocalPipelineConfiguration(hopGui.getShell(), prcSerializer);
            }
            // Then the local workflow runconfig
            // 
            IHopMetadataSerializer<WorkflowRunConfiguration> wrcSerializer = hopGui.getMetadataProvider().getSerializer(WorkflowRunConfiguration.class);
            localFound = false;
            List<WorkflowRunConfiguration> workflowRunConfigs = wrcSerializer.loadAll();
            for (WorkflowRunConfiguration workflowRunConfig : workflowRunConfigs) {
                if (workflowRunConfig.getEngineRunConfiguration() instanceof LocalWorkflowRunConfiguration) {
                    localFound = true;
                }
            }
            if (!localFound) {
                MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
                box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfig.Dialog.Header"));
                box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfig.Dialog.Message"));
                int anwser = box.open();
                if ((anwser & SWT.YES) != 0) {
                    LocalWorkflowRunConfiguration localWorkflowRunConfiguration = new LocalWorkflowRunConfiguration();
                    localWorkflowRunConfiguration.setEnginePluginId("Local");
                    WorkflowRunConfiguration local = new WorkflowRunConfiguration("local", BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfigDescription.Text"), localWorkflowRunConfiguration);
                    wrcSerializer.save(local);
                }
            }
            // Ask to put the project in a lifecycle environment
            // 
            MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
            box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Header"));
            box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Message1") + Const.CR + BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Message2"));
            int anwser = box.open();
            if ((anwser & SWT.YES) != 0) {
                addNewEnvironment();
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "ProjectGuiPlugin.AddProject.Error.Dialog.Header"), BaseMessages.getString(PKG, "ProjectGuiPlugin.AddProject.Error.Dialog.Message"), e);
    }
}

Example 2 with PipelineRunConfiguration

use of org.apache.hop.pipeline.config.PipelineRunConfiguration in project hop by apache.

the class MainBeam method main.

public static void main(String[] args) {
    try {
        System.out.println("Argument 1 : Pipeline filename (.hpl)   : " + args[0]);
        System.out.println("Argument 2 : Metadata filename (.json)  : " + args[1]);
        System.out.println("Argument 3 : Pipeline run configuration : " + args[2]);
        System.out.println(">>>>>> Initializing Hop...");
        HopEnvironment.init();
        // Read the pipeline XML and metadata JSON (optionally from Hadoop FS)
        // 
        String pipelineMetaXml = readFileIntoString(args[0], "UTF-8");
        String metadataJson = readFileIntoString(args[1], "UTF-8");
        String runConfigName = args[2];
        // Inflate the metadata:
        // 
        SerializableMetadataProvider metadataProvider = new SerializableMetadataProvider(metadataJson);
        // Load the pipeline run configuration from this metadata provider:
        // 
        IHopMetadataSerializer<PipelineRunConfiguration> serializer = metadataProvider.getSerializer(PipelineRunConfiguration.class);
        if (!serializer.exists(runConfigName)) {
            throw new HopException("The specified pipeline run configuration '" + runConfigName + "' doesn't exist");
        }
        System.out.println(">>>>>> Loading pipeline metadata");
        PipelineMeta pipelineMeta = new PipelineMeta(XmlHandler.loadXmlString(pipelineMetaXml, PipelineMeta.XML_TAG), metadataProvider);
        System.out.println(">>>>>> Building Apache Beam Pipeline...");
        PluginRegistry registry = PluginRegistry.getInstance();
        IPlugin beamInputPlugin = registry.getPlugin(TransformPluginType.class, BeamConst.STRING_BEAM_INPUT_PLUGIN_ID);
        if (beamInputPlugin != null) {
            System.out.println(">>>>>> Found Beam Input transform plugin class loader");
        } else {
            throw new HopException("ERROR: Unable to find Beam Input transform plugin. Is it in the fat jar? ");
        }
        IVariables variables = Variables.getADefaultVariableSpace();
        // Execute it...
        // 
        IPipelineEngine<PipelineMeta> pipelineEngine = PipelineEngineFactory.createPipelineEngine(variables, runConfigName, metadataProvider, pipelineMeta);
        System.out.println(">>>>>> Pipeline executing starting...");
        pipelineEngine.execute();
        pipelineEngine.waitUntilFinished();
        System.out.println(">>>>>> Execution finished...");
        System.exit(0);
    } catch (Exception e) {
        System.err.println("Error running Beam pipeline: " + e.getMessage());
        e.printStackTrace();
        System.exit(1);
    }
}

Example 3 with PipelineRunConfiguration

use of org.apache.hop.pipeline.config.PipelineRunConfiguration in project hop by apache.

the class BeamDirectPipelineEngineTest method testDirectPipelineEngine.

@Test
public void testDirectPipelineEngine() throws Exception {
    IPipelineEngineRunConfiguration configuration = new BeamDirectPipelineRunConfiguration();
    configuration.setEnginePluginId("BeamDirectPipelineEngine");
    PipelineRunConfiguration pipelineRunConfiguration = new PipelineRunConfiguration("direct", "description", Arrays.asList(new VariableValueDescription("VAR1", "value1", "description1")), configuration);
    metadataProvider.getSerializer(PipelineRunConfiguration.class).save(pipelineRunConfiguration);
    PipelineMeta pipelineMeta = BeamPipelineMetaUtil.generateBeamInputOutputPipelineMeta("input-process-output", "INPUT", "OUTPUT", metadataProvider);
    IPipelineEngine<PipelineMeta> engine = createAndExecutePipeline(pipelineRunConfiguration.getName(), metadataProvider, pipelineMeta);
    validateInputOutputEngineMetrics(engine);
    assertEquals("value1", engine.getVariable("VAR1"));
}

Example 4 with PipelineRunConfiguration

use of org.apache.hop.pipeline.config.PipelineRunConfiguration in project hop by apache.

the class ActionPipelineGuiPlugin method copyAsActionToClipboard.

@GuiContextAction(id = "pipeline-graph-transform-10300-copy-pipeline-action", parentId = HopGuiPipelineContext.CONTEXT_ID, type = GuiActionType.Modify, name = "Copy as pipeline action", tooltip = "Copy this pipeline as an action so you can paste it in a workflow", image = "ui/images/copy.svg", category = "Basic", categoryOrder = "1")
public void copyAsActionToClipboard(HopGuiPipelineContext context) {
    PipelineMeta pipelineMeta = context.getPipelineMeta();
    HopGuiPipelineGraph pipelineGraph = context.getPipelineGraph();
    IVariables variables = pipelineGraph.getVariables();
    ActionPipeline actionPipeline = new ActionPipeline(pipelineMeta.getName());
    HopGuiFileOpenedExtension ext = new HopGuiFileOpenedExtension(null, variables, pipelineMeta.getFilename());
    // 
    try {
        ExtensionPointHandler.callExtensionPoint(LogChannel.UI, variables, HopGuiExtensionPoint.HopGuiFileOpenedDialog.id, ext);
    } catch (Exception xe) {
        LogChannel.UI.logError("Error handling extension point 'HopGuiFileOpenDialog'", xe);
    }
    actionPipeline.setFileName(ext.filename);
    // 
    try {
        IHopMetadataProvider metadataProvider = pipelineGraph.getHopGui().getMetadataProvider();
        IHopMetadataSerializer<PipelineRunConfiguration> serializer = metadataProvider.getSerializer(PipelineRunConfiguration.class);
        List<String> configNames = serializer.listObjectNames();
        if (!configNames.isEmpty()) {
            if (configNames.size() == 1) {
                actionPipeline.setRunConfiguration(configNames.get(0));
            } else {
                EnterSelectionDialog dialog = new EnterSelectionDialog(pipelineGraph.getShell(), configNames.toArray(new String[0]), "Select run configuration", "Select the pipeline run configuration to use in the action:");
                String configName = dialog.open();
                if (configName != null) {
                    actionPipeline.setRunConfiguration(configName);
                }
            }
        }
    } catch (Exception e) {
        new ErrorDialog(pipelineGraph.getShell(), "Error", "Error selecting pipeline run configurations", e);
    }
    ActionMeta actionMeta = new ActionMeta(actionPipeline);
    StringBuilder xml = new StringBuilder(5000).append(XmlHandler.getXmlHeader());
    xml.append(XmlHandler.openTag(HopGuiWorkflowClipboardDelegate.XML_TAG_WORKFLOW_ACTIONS)).append(Const.CR);
    xml.append(XmlHandler.openTag(HopGuiWorkflowClipboardDelegate.XML_TAG_ACTIONS)).append(Const.CR);
    xml.append(actionMeta.getXml());
    xml.append(XmlHandler.closeTag(HopGuiWorkflowClipboardDelegate.XML_TAG_ACTIONS)).append(Const.CR);
    xml.append(XmlHandler.closeTag(HopGuiWorkflowClipboardDelegate.XML_TAG_WORKFLOW_ACTIONS)).append(Const.CR);
    pipelineGraph.pipelineClipboardDelegate.toClipboard(xml.toString());
}

Example 5 with PipelineRunConfiguration

use of org.apache.hop.pipeline.config.PipelineRunConfiguration in project hop by apache.

the class MetaInject method createInjectPipeline.

Pipeline createInjectPipeline() throws HopException {
    LocalPipelineEngine lpe = new LocalPipelineEngine(data.pipelineMeta, this, this);
    if (!Utils.isEmpty(meta.getRunConfigurationName())) {
        PipelineRunConfiguration prc = metadataProvider.getSerializer(PipelineRunConfiguration.class).load(meta.getRunConfigurationName());
        lpe.setPipelineRunConfiguration(prc);
    }
    return lpe;
}