Examples with LocalPipelineRunConfiguration org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration used on opensource projects

Example 1 with LocalPipelineRunConfiguration

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration in project hop by apache.

the class ProjectsGuiPlugin method addNewProject.

@GuiToolbarElement(root = HopGui.ID_MAIN_TOOLBAR, id = ID_TOOLBAR_PROJECT_ADD, toolTip = "i18n::HopGui.Toolbar.Project.Add.Tooltip", image = "project-add.svg")
public void addNewProject() {
    HopGui hopGui = HopGui.getInstance();
    IVariables variables = hopGui.getVariables();
    try {
        ProjectsConfig config = ProjectsConfigSingleton.getConfig();
        String standardProjectsFolder = variables.resolve(config.getStandardProjectsFolder());
        String defaultProjectConfigFilename = variables.resolve(config.getDefaultProjectConfigFile());
        ProjectConfig projectConfig = new ProjectConfig("", standardProjectsFolder, defaultProjectConfigFilename);
        Project project = new Project();
        project.setParentProjectName(config.getStandardParentProject());
        ProjectDialog projectDialog = new ProjectDialog(hopGui.getShell(), project, projectConfig, variables, false);
        String projectName = projectDialog.open();
        if (projectName != null) {
            config.addProjectConfig(projectConfig);
            HopConfig.getInstance().saveToFile();
            // Save the project-config.json file as well in the project itself
            // 
            FileObject configFile = HopVfs.getFileObject(projectConfig.getActualProjectConfigFilename(variables));
            if (!configFile.exists()) {
                // Create the empty configuration file if it does not exists
                project.saveToFile();
            } else {
                // If projects exists load configuration
                MessageBox box = new MessageBox(hopGui.getShell(), SWT.ICON_QUESTION | SWT.OK);
                box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.ProjectExists.Dialog.Header"));
                box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.ProjectExists.Dialog.Message"));
                box.open();
                project.readFromFile();
            }
            refreshProjectsList();
            selectProjectInList(projectName);
            enableHopGuiProject(projectName, project, null);
            // Now see if these project contains any local run configurations.
            // If not we can add those automatically
            // 
            // First pipeline
            // 
            IHopMetadataSerializer<PipelineRunConfiguration> prcSerializer = hopGui.getMetadataProvider().getSerializer(PipelineRunConfiguration.class);
            List<PipelineRunConfiguration> pipelineRunConfigs = prcSerializer.loadAll();
            boolean localFound = false;
            for (PipelineRunConfiguration pipelineRunConfig : pipelineRunConfigs) {
                if (pipelineRunConfig.getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration) {
                    localFound = true;
                }
            }
            if (!localFound) {
                PipelineExecutionConfigurationDialog.createLocalPipelineConfiguration(hopGui.getShell(), prcSerializer);
            }
            // Then the local workflow runconfig
            // 
            IHopMetadataSerializer<WorkflowRunConfiguration> wrcSerializer = hopGui.getMetadataProvider().getSerializer(WorkflowRunConfiguration.class);
            localFound = false;
            List<WorkflowRunConfiguration> workflowRunConfigs = wrcSerializer.loadAll();
            for (WorkflowRunConfiguration workflowRunConfig : workflowRunConfigs) {
                if (workflowRunConfig.getEngineRunConfiguration() instanceof LocalWorkflowRunConfiguration) {
                    localFound = true;
                }
            }
            if (!localFound) {
                MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
                box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfig.Dialog.Header"));
                box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfig.Dialog.Message"));
                int anwser = box.open();
                if ((anwser & SWT.YES) != 0) {
                    LocalWorkflowRunConfiguration localWorkflowRunConfiguration = new LocalWorkflowRunConfiguration();
                    localWorkflowRunConfiguration.setEnginePluginId("Local");
                    WorkflowRunConfiguration local = new WorkflowRunConfiguration("local", BaseMessages.getString(PKG, "ProjectGuiPlugin.LocalWFRunConfigDescription.Text"), localWorkflowRunConfiguration);
                    wrcSerializer.save(local);
                }
            }
            // Ask to put the project in a lifecycle environment
            // 
            MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
            box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Header"));
            box.setMessage(BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Message1") + Const.CR + BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Message2"));
            int anwser = box.open();
            if ((anwser & SWT.YES) != 0) {
                addNewEnvironment();
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "ProjectGuiPlugin.AddProject.Error.Dialog.Header"), BaseMessages.getString(PKG, "ProjectGuiPlugin.AddProject.Error.Dialog.Message"), e);
    }
}

Example 2 with LocalPipelineRunConfiguration

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration in project hop by apache.

the class HopGuiPipelineGraph method addRowsSamplerToPipeline.

private void addRowsSamplerToPipeline(IPipelineEngine<PipelineMeta> pipeline) {
    if (!(pipeline.getPipelineRunConfiguration().getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration)) {
        return;
    }
    LocalPipelineRunConfiguration lprConfig = (LocalPipelineRunConfiguration) pipeline.getPipelineRunConfiguration().getEngineRunConfiguration();
    if (StringUtils.isEmpty(lprConfig.getSampleTypeInGui())) {
        return;
    }
    try {
        SampleType sampleType = SampleType.valueOf(lprConfig.getSampleTypeInGui());
        if (sampleType == SampleType.None) {
            return;
        }
        final int sampleSize = Const.toInt(pipeline.resolve(lprConfig.getSampleSize()), 100);
        if (sampleSize <= 0) {
            return;
        }
        outputRowsMap = new HashMap<>();
        final Random random = new Random();
        for (final String transformName : pipelineMeta.getTransformNames()) {
            IEngineComponent component = pipeline.findComponent(transformName, 0);
            if (component != null) {
                component.addRowListener(new RowAdapter() {

                    int nrRows = 0;

                    @Override
                    public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                        RowBuffer rowBuffer = outputRowsMap.get(transformName);
                        if (rowBuffer == null) {
                            rowBuffer = new RowBuffer(rowMeta);
                            outputRowsMap.put(transformName, rowBuffer);
                            // 
                            if (sampleType == SampleType.Last) {
                                rowBuffer.setBuffer(Collections.synchronizedList(new LinkedList<>()));
                            } else {
                                rowBuffer.setBuffer(Collections.synchronizedList(new ArrayList<>()));
                            }
                        }
                        // 
                        if (sampleType != SampleType.None) {
                            try {
                                row = rowMeta.cloneRow(row);
                            } catch (HopValueException e) {
                                throw new HopTransformException("Error copying row for preview purposes", e);
                            }
                        }
                        switch(sampleType) {
                            case First:
                                {
                                    if (rowBuffer.size() < sampleSize) {
                                        rowBuffer.addRow(row);
                                    }
                                }
                                break;
                            case Last:
                                {
                                    rowBuffer.addRow(0, row);
                                    if (rowBuffer.size() > sampleSize) {
                                        rowBuffer.removeRow(rowBuffer.size() - 1);
                                    }
                                }
                                break;
                            case Random:
                                {
                                    // Reservoir sampling
                                    // 
                                    nrRows++;
                                    if (rowBuffer.size() < sampleSize) {
                                        rowBuffer.addRow(row);
                                    } else {
                                        int randomIndex = random.nextInt(nrRows);
                                        if (randomIndex < sampleSize) {
                                            rowBuffer.setRow(randomIndex, row);
                                        }
                                    }
                                }
                                break;
                        }
                    }
                });
            }
        }
    } catch (Exception e) {
    // Ignore : simply not recognized or empty
    }
}

Example 3 with LocalPipelineRunConfiguration

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration in project hop by apache.

the class PipelineExecutionConfigurationDialog method createLocalPipelineConfiguration.

public static final String createLocalPipelineConfiguration(Shell shell, IHopMetadataSerializer<PipelineRunConfiguration> prcSerializer) {
    try {
        MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
        box.setText(BaseMessages.getString(PKG, "PipelineExecutionConfigurationDialog.NoRunConfigurationDefined.Title"));
        box.setMessage(BaseMessages.getString(PKG, "PipelineExecutionConfigurationDialog.NoRunConfigurationDefined.Message"));
        int answer = box.open();
        if ((answer & SWT.YES) != 0) {
            LocalPipelineRunConfiguration localPipelineRunConfiguration = new LocalPipelineRunConfiguration();
            localPipelineRunConfiguration.setEnginePluginId("Local");
            PipelineRunConfiguration local = new PipelineRunConfiguration("local", BaseMessages.getString(PKG, "PipelineExecutionConfigurationDialog.LocalRunConfiguration.Description"), new ArrayList<>(), localPipelineRunConfiguration);
            prcSerializer.save(local);
            return local.getName();
        }
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "PipelineExecutionConfigurationDialog.ErrorSavingRunConfiguration.Title"), BaseMessages.getString(PKG, "PipelineExecutionConfigurationDialog.ErrorSavingRunConfiguration.Message"), e);
    }
    return null;
}

Example 4 with LocalPipelineRunConfiguration

use of org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration in project hop by apache.

the class HopGuiPipelineGraph method showTransformOutputData.

private boolean showTransformOutputData(TransformMeta dataTransformMeta, RowBuffer rowBuffer) {
    if (rowBuffer != null) {
        synchronized (rowBuffer.getBuffer()) {
            if (!rowBuffer.isEmpty()) {
                try {
                    String title = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Header", dataTransformMeta.getName());
                    String message = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.OutputRows.Text", dataTransformMeta.getName());
                    String prefix = "";
                    if (pipeline != null && pipeline.getPipelineRunConfiguration() != null) {
                        PipelineRunConfiguration pipelineRunConfiguration = pipeline.getPipelineRunConfiguration();
                        if (pipelineRunConfiguration.getEngineRunConfiguration() instanceof LocalPipelineRunConfiguration) {
                            String sampleTypeInGui = ((LocalPipelineRunConfiguration) pipelineRunConfiguration.getEngineRunConfiguration()).getSampleTypeInGui();
                            if (StringUtils.isNotEmpty(sampleTypeInGui)) {
                                try {
                                    SampleType sampleType = SampleType.valueOf(sampleTypeInGui);
                                    switch(sampleType) {
                                        case None:
                                            break;
                                        case First:
                                            prefix = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.First.Text");
                                            break;
                                        case Last:
                                            prefix = BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Last.Text");
                                            break;
                                        case Random:
                                            prefix += BaseMessages.getString(PKG, "PipelineGraph.ViewOutput.OutputDialog.Random.Text");
                                            ;
                                            break;
                                        default:
                                            break;
                                    }
                                } catch (Exception ex) {
                                    LogChannel.UI.logError("Unknown sample type: " + sampleTypeInGui);
                                }
                            }
                        }
                    }
                    PreviewRowsDialog previewRowsDialog = new PreviewRowsDialog(hopGui.getShell(), variables, SWT.NONE, dataTransformMeta.getName(), rowBuffer.getRowMeta(), rowBuffer.getBuffer());
                    previewRowsDialog.setTitleMessage(title, prefix + message);
                    previewRowsDialog.open();
                } catch (Exception ex) {
                    new ErrorDialog(hopGui.getShell(), "Error", "Error showing preview dialog", ex);
                }
            }
        }
        return true;
    }
    return false;
}