Examples with PipelineDataProbe org.apache.hop.reflection.probe.transform.PipelineDataProbe used on opensource projects

Example 1 with PipelineDataProbe

use of org.apache.hop.reflection.probe.transform.PipelineDataProbe in project hop by apache.

the class PipelineDataProbeXp method executeProbingPipeline.

/**
 * Execute a probing pipeline for the current pipeline. Add a listener to the transform copies.
 * Send the data to the PipelineDataProbe transform(s) in the probing pipeline
 *
 * @param pipelineProbe
 * @param dataProbeLocation
 * @param loggingPipelineFilename The pipeline to start for the location
 * @param pipeline The parent pipeline to listen to
 * @param variables
 * @throws HopException
 */
private synchronized void executeProbingPipeline(PipelineProbe pipelineProbe, DataProbeLocation dataProbeLocation, String loggingPipelineFilename, IPipelineEngine<PipelineMeta> pipeline, IVariables variables) throws HopException {
    PipelineMeta probingPipelineMeta = new PipelineMeta(loggingPipelineFilename, pipeline.getMetadataProvider(), true, variables);
    // Create a local pipeline engine...
    // 
    LocalPipelineEngine probingPipeline = new LocalPipelineEngine(probingPipelineMeta, variables, pipeline);
    // Flag it as a probing and logging pipeline so we don't try to probe or log ourselves...
    // 
    probingPipeline.getExtensionDataMap().put(PIPELINE_DATA_PROBE_FLAG, "Y");
    probingPipeline.getExtensionDataMap().put(PipelineStartLoggingXp.PIPELINE_LOGGING_FLAG, "Y");
    // Only log errors
    // 
    probingPipeline.setLogLevel(LogLevel.ERROR);
    probingPipeline.prepareExecution();
    List<IEngineComponent> componentCopies = pipeline.getComponentCopies(dataProbeLocation.getSourceTransformName());
    for (IEngineComponent componentCopy : componentCopies) {
        // 
        for (TransformMetaDataCombi combi : probingPipeline.getTransforms()) {
            if (combi.transform instanceof PipelineDataProbe) {
                // Give the transform a bit more information to work with...
                // 
                PipelineDataProbe pipelineDataProbe = (PipelineDataProbe) combi.transform;
                pipelineDataProbe.setSourcePipelineName(pipeline.getPipelineMeta().getName());
                pipelineDataProbe.setSourceTransformLogChannelId(pipeline.getLogChannelId());
                pipelineDataProbe.setSourceTransformName(componentCopy.getName());
                pipelineDataProbe.setSourceTransformCopy(componentCopy.getCopyNr());
                try {
                    final RowProducer rowProducer = probingPipeline.addRowProducer(combi.transformName, combi.copy);
                    // For every copy of the component, add an input row set to the parent pipeline...
                    // 
                    componentCopy.addRowListener(new RowAdapter() {

                        @Override
                        public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            // Pass this row to the row producer...
                            // 
                            rowProducer.putRow(rowMeta, row);
                        }
                    });
                    // If the pipeline we're the transform is and we can safely stop streaming...
                    // 
                    pipeline.addExecutionFinishedListener(pe -> rowProducer.finished());
                } catch (HopException e) {
                    throw new HopTransformException("Error adding row producer to transform '" + combi.transformName + "'", e);
                }
            }
        }
    }
    // Execute the logging pipeline to save the logging information
    // 
    probingPipeline.startThreads();
    // We'll not wait around until this is finished...
    // The pipeline should stop automatically when the parent does
    // 
    pipeline.addExecutionStoppedListener(e -> probingPipeline.stopAll());
}