Examples with IPipelineEngineRunConfiguration org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration used on opensource projects

Example 1 with IPipelineEngineRunConfiguration

use of org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration in project hop by apache.

the class PipelineEngineFactory method createPipelineEngine.

public static final <T extends PipelineMeta> IPipelineEngine<T> createPipelineEngine(PipelineRunConfiguration pipelineRunConfiguration, T pipelineMeta) throws HopException {
    IPipelineEngineRunConfiguration engineRunConfiguration = pipelineRunConfiguration.getEngineRunConfiguration();
    if (engineRunConfiguration == null) {
        throw new HopException("There is no pipeline execution engine specified in run configuration '" + pipelineRunConfiguration.getName() + "'");
    }
    String enginePluginId = engineRunConfiguration.getEnginePluginId();
    // Load this engine from the plugin registry
    // 
    PluginRegistry pluginRegistry = PluginRegistry.getInstance();
    IPlugin plugin = pluginRegistry.findPluginWithId(PipelineEnginePluginType.class, enginePluginId);
    if (plugin == null) {
        throw new HopException("Unable to find pipeline engine plugin type with ID '" + enginePluginId + "'");
    }
    IPipelineEngine<T> pipelineEngine = pluginRegistry.loadClass(plugin, IPipelineEngine.class);
    pipelineEngine.setPipelineRunConfiguration(pipelineRunConfiguration);
    // Apply the variables in the pipeline run configuration
    // 
    applyVariableDefinitions(pipelineEngine, pipelineRunConfiguration.getConfigurationVariables());
    pipelineEngine.setPipelineMeta(pipelineMeta);
    return pipelineEngine;
}

Example 2 with IPipelineEngineRunConfiguration

use of org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration in project hop by apache.

the class BeamPipelineEngine method prepareExecution.

@Override
public void prepareExecution() throws HopException {
    ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        executionStartDate = new Date();
        // Explain to various classes in the Beam API (@see org.apache.beam.sdk.io.FileSystems)
        // what the context classloader is.
        // Set it back when we're done here.
        // 
        Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
        setPreparing(true);
        IPipelineEngineRunConfiguration engineRunConfiguration = pipelineRunConfiguration.getEngineRunConfiguration();
        validatePipelineRunConfigurationClass(engineRunConfiguration);
        if (!(engineRunConfiguration instanceof IBeamPipelineEngineRunConfiguration)) {
            throw new HopException("A beam pipeline needs a beam pipeline engine configuration to run, not '" + pipelineRunConfiguration.getName() + "'");
        }
        if (metadataProvider == null) {
            throw new HopException("The beam pipeline engine didn't receive a metadata");
        }
        beamEngineRunConfiguration = (IBeamPipelineEngineRunConfiguration) engineRunConfiguration;
        converter = new HopPipelineMetaToBeamPipelineConverter(this, pipelineMeta, metadataProvider, beamEngineRunConfiguration);
        beamPipeline = converter.createPipeline();
        FileSystems.setDefaultPipelineOptions(beamPipeline.getOptions());
        // Create a new log channel when we start the action
        // It's only now that we use it
        // 
        logChannel.logBasic("Executing this pipeline using the Beam Pipeline Engine with run configuration '" + pipelineRunConfiguration.getName() + "'");
        PipelineExecutionConfiguration pipelineExecutionConfiguration = new PipelineExecutionConfiguration();
        pipelineExecutionConfiguration.setRunConfiguration(pipelineRunConfiguration.getName());
        if (logLevel != null) {
            pipelineExecutionConfiguration.setLogLevel(logLevel);
        }
        if (previousResult != null) {
            pipelineExecutionConfiguration.setPreviousResult(previousResult);
        }
        setRunning(false);
        setReadyToStart(true);
    } catch (Exception e) {
        setRunning(false);
        setReadyToStart(false);
        setStopped(true);
        setErrors(getErrors() + 1);
        setPaused(false);
        setPreparing(false);
        throw new HopException("Error preparing remote pipeline", e);
    } finally {
        setPreparing(false);
        Thread.currentThread().setContextClassLoader(oldContextClassLoader);
    }
}

Example 3 with IPipelineEngineRunConfiguration

use of org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration in project hop by apache.

the class BeamDirectPipelineEngineTest method testDirectPipelineEngine.

@Test
public void testDirectPipelineEngine() throws Exception {
    IPipelineEngineRunConfiguration configuration = new BeamDirectPipelineRunConfiguration();
    configuration.setEnginePluginId("BeamDirectPipelineEngine");
    PipelineRunConfiguration pipelineRunConfiguration = new PipelineRunConfiguration("direct", "description", Arrays.asList(new VariableValueDescription("VAR1", "value1", "description1")), configuration);
    metadataProvider.getSerializer(PipelineRunConfiguration.class).save(pipelineRunConfiguration);
    PipelineMeta pipelineMeta = BeamPipelineMetaUtil.generateBeamInputOutputPipelineMeta("input-process-output", "INPUT", "OUTPUT", metadataProvider);
    IPipelineEngine<PipelineMeta> engine = createAndExecutePipeline(pipelineRunConfiguration.getName(), metadataProvider, pipelineMeta);
    validateInputOutputEngineMetrics(engine);
    assertEquals("value1", engine.getVariable("VAR1"));
}

Example 4 with IPipelineEngineRunConfiguration

use of org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration in project hop by apache.

the class PipelineRunConfigurationEditor method changeWorkingEngineConfiguration.

private void changeWorkingEngineConfiguration(PipelineRunConfiguration meta) {
    String pluginName = wPluginType.getText();
    IPipelineEngineRunConfiguration engineRunConfiguration = metaMap.get(pluginName);
    if (engineRunConfiguration != null) {
        // Switch to the right plugin type
        // 
        meta.setEngineRunConfiguration(engineRunConfiguration);
    } else {
        meta.setEngineRunConfiguration(null);
    }
}

Example 5 with IPipelineEngineRunConfiguration

use of org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration in project hop by apache.

the class PipelineRunConfigurationEditor method populateMetaMap.

private Map<String, IPipelineEngineRunConfiguration> populateMetaMap() {
    metaMap = new HashMap<>();
    List<IPlugin> plugins = PluginRegistry.getInstance().getPlugins(PipelineEnginePluginType.class);
    for (IPlugin plugin : plugins) {
        try {
            IPipelineEngine<?> engine = PluginRegistry.getInstance().loadClass(plugin, IPipelineEngine.class);
            // Get the default run configuration for the engine.
            // 
            IPipelineEngineRunConfiguration engineRunConfiguration = engine.createDefaultPipelineEngineRunConfiguration();
            engineRunConfiguration.setEnginePluginId(plugin.getIds()[0]);
            engineRunConfiguration.setEnginePluginName(plugin.getName());
            metaMap.put(engineRunConfiguration.getEnginePluginName(), engineRunConfiguration);
        } catch (Exception e) {
            HopGui.getInstance().getLog().logError("Error instantiating pipeline run configuration plugin", e);
        }
    }
    return metaMap;
}