Examples with HopPipelineMetaToBeamPipelineConverter org.apache.hop.beam.pipeline.HopPipelineMetaToBeamPipelineConverter used on opensource projects

Example 1 with HopPipelineMetaToBeamPipelineConverter

use of org.apache.hop.beam.pipeline.HopPipelineMetaToBeamPipelineConverter in project hop by apache.

the class BeamPipelineEngine method prepareExecution.

@Override
public void prepareExecution() throws HopException {
    ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        executionStartDate = new Date();
        // Explain to various classes in the Beam API (@see org.apache.beam.sdk.io.FileSystems)
        // what the context classloader is.
        // Set it back when we're done here.
        // 
        Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
        setPreparing(true);
        IPipelineEngineRunConfiguration engineRunConfiguration = pipelineRunConfiguration.getEngineRunConfiguration();
        validatePipelineRunConfigurationClass(engineRunConfiguration);
        if (!(engineRunConfiguration instanceof IBeamPipelineEngineRunConfiguration)) {
            throw new HopException("A beam pipeline needs a beam pipeline engine configuration to run, not '" + pipelineRunConfiguration.getName() + "'");
        }
        if (metadataProvider == null) {
            throw new HopException("The beam pipeline engine didn't receive a metadata");
        }
        beamEngineRunConfiguration = (IBeamPipelineEngineRunConfiguration) engineRunConfiguration;
        converter = new HopPipelineMetaToBeamPipelineConverter(this, pipelineMeta, metadataProvider, beamEngineRunConfiguration);
        beamPipeline = converter.createPipeline();
        FileSystems.setDefaultPipelineOptions(beamPipeline.getOptions());
        // Create a new log channel when we start the action
        // It's only now that we use it
        // 
        logChannel.logBasic("Executing this pipeline using the Beam Pipeline Engine with run configuration '" + pipelineRunConfiguration.getName() + "'");
        PipelineExecutionConfiguration pipelineExecutionConfiguration = new PipelineExecutionConfiguration();
        pipelineExecutionConfiguration.setRunConfiguration(pipelineRunConfiguration.getName());
        if (logLevel != null) {
            pipelineExecutionConfiguration.setLogLevel(logLevel);
        }
        if (previousResult != null) {
            pipelineExecutionConfiguration.setPreviousResult(previousResult);
        }
        setRunning(false);
        setReadyToStart(true);
    } catch (Exception e) {
        setRunning(false);
        setReadyToStart(false);
        setStopped(true);
        setErrors(getErrors() + 1);
        setPaused(false);
        setPreparing(false);
        throw new HopException("Error preparing remote pipeline", e);
    } finally {
        setPreparing(false);
        Thread.currentThread().setContextClassLoader(oldContextClassLoader);
    }
}

Example 2 with HopPipelineMetaToBeamPipelineConverter

use of org.apache.hop.beam.pipeline.HopPipelineMetaToBeamPipelineConverter in project hop by apache.

the class PipelineTestBase method createRunPipeline.

@Ignore
public void createRunPipeline(IVariables variables, PipelineMeta pipelineMeta) throws Exception {
    /*
    FileOutputStream fos = new FileOutputStream( "/tmp/"+pipelineMeta.getName()+".hpl" );
    fos.write( pipelineMeta.getXml().getBytes() );
    fos.close();
    */
    PipelineOptions pipelineOptions = PipelineOptionsFactory.create();
    pipelineOptions.setJobName(pipelineMeta.getName());
    pipelineOptions.setUserAgent(BeamConst.STRING_HOP_BEAM);
    BeamDirectPipelineRunConfiguration beamRunConfig = new BeamDirectPipelineRunConfiguration();
    beamRunConfig.setTempLocation(System.getProperty("java.io.tmpdir"));
    // No extra plugins to load : null option
    HopPipelineMetaToBeamPipelineConverter converter = new HopPipelineMetaToBeamPipelineConverter(variables, pipelineMeta, metadataProvider, beamRunConfig);
    Pipeline pipeline = converter.createPipeline();
    PipelineResult pipelineResult = pipeline.run();
    pipelineResult.waitUntilFinish();
    MetricResults metricResults = pipelineResult.metrics();
    MetricQueryResults allResults = metricResults.queryMetrics(MetricsFilter.builder().build());
    for (MetricResult<Long> result : allResults.getCounters()) {
        System.out.println("Name: " + result.getName() + " Attempted: " + result.getAttempted());
    }
}