Examples with LocalPipelineEngine org.apache.hop.pipeline.engines.local.LocalPipelineEngine used on opensource projects

Example 26 with LocalPipelineEngine

use of org.apache.hop.pipeline.engines.local.LocalPipelineEngine in project hop by apache.

the class TestUtilities method loadAndRunPipeline.

public static Pipeline loadAndRunPipeline(String path, Object... parameters) throws Exception {
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setPipelineType(PipelineMeta.PipelineType.Normal);
    Pipeline trans = new LocalPipelineEngine(pipelineMeta);
    if (parameters != null) {
        if (parameters.length % 2 == 1) {
            throw new IllegalArgumentException("Parameters should be an array of pairs 'parameter'-'value'-...");
        }
        for (int i = 0; i < parameters.length; i += 2) {
            Object parameter = parameters[i];
            Object value = parameters[i + 1];
            trans.setParameterValue(parameter.toString(), value.toString());
        }
    }
    trans.prepareExecution();
    trans.startThreads();
    trans.waitUntilFinished();
    return trans;
}

Example 27 with LocalPipelineEngine

use of org.apache.hop.pipeline.engines.local.LocalPipelineEngine in project hop by apache.

the class PipelineTestFactory method executeTestTransformation.

public static List<RowMetaAndData> executeTestTransformation(PipelineMeta pipelineMeta, String injectorTransformName, String testTransformName, String dummyTransformName, List<RowMetaAndData> inputData) throws HopException {
    // Now execute the pipeline...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    // Capture the rows that come out of the dummy transform...
    // 
    ITransform si = pipeline.getTransform(dummyTransformName, 0);
    RowTransformCollector dummyRc = new RowTransformCollector();
    si.addRowListener(dummyRc);
    // Add a row producer...
    // 
    RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
    // Start the transforms...
    // 
    pipeline.startThreads();
    // Inject the actual test rows...
    // 
    List<RowMetaAndData> inputList = inputData;
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    // Wait until the pipeline is finished...
    // 
    pipeline.waitUntilFinished();
    // 
    if (pipeline.getResult().getNrErrors() > 0) {
        throw new HopException("Test pipeline finished with errors. Check the log.");
    }
    // 
    return dummyRc.getRowsRead();
}

Example 28 with LocalPipelineEngine

use of org.apache.hop.pipeline.engines.local.LocalPipelineEngine in project hop by apache.

the class PipelineTestFactory method executeTestTransformationError.

public static Map<String, RowTransformCollector> executeTestTransformationError(PipelineMeta pipelineMeta, String injectorTransformName, String testTransformName, String dummyTransformName, String errorTransformName, List<RowMetaAndData> inputData) throws HopException {
    // Now execute the pipeline...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    // Capture the rows that come out of the dummy transform...
    // 
    ITransform si = pipeline.getTransform(dummyTransformName, 0);
    RowTransformCollector dummyRc = new RowTransformCollector();
    si.addRowListener(dummyRc);
    ITransform junit = pipeline.getTransform(testTransformName, 0);
    RowTransformCollector dummyJu = new RowTransformCollector();
    junit.addRowListener(dummyJu);
    // add error handler
    ITransform er = pipeline.getTransform(errorTransformName, 0);
    RowTransformCollector erColl = new RowTransformCollector();
    er.addRowListener(erColl);
    // Add a row producer...
    // 
    RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
    // Start the transforms...
    // 
    pipeline.startThreads();
    // Inject the actual test rows...
    // 
    List<RowMetaAndData> inputList = inputData;
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    // Wait until the pipeline is finished...
    // 
    pipeline.waitUntilFinished();
    // 
    if (pipeline.getResult().getNrErrors() > 0) {
        throw new HopException("Test pipeline finished with errors. Check the log.");
    }
    // Return the result from the dummy transform...
    Map<String, RowTransformCollector> ret = new HashMap<>();
    ret.put(dummyTransformName, dummyRc);
    ret.put(errorTransformName, erColl);
    ret.put(testTransformName, dummyJu);
    return ret;
}

Example 29 with LocalPipelineEngine

use of org.apache.hop.pipeline.engines.local.LocalPipelineEngine in project hop by apache.

the class GetXMLDataTest method testGetXMLDataSimple1.

/**
 * Test case for Get XML Data transform, very simple example.
 *
 * @throws Exception Upon any exception
 */
public void testGetXMLDataSimple1() throws Exception {
    HopEnvironment.init();
    // 
    // Create a new pipeline...
    // 
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setName("getxmldata1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector transform...
    // 
    String injectorTransformName = "injector transform";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(TransformPluginType.class, im);
    TransformMeta injectorTransform = new TransformMeta(injectorPid, injectorTransformName, im);
    pipelineMeta.addTransform(injectorTransform);
    // 
    // Create a Get XML Data transform
    // 
    String getXMLDataName = "get xml data transform";
    GetXmlDataMeta gxdm = new GetXmlDataMeta();
    String getXMLDataPid = registry.getPluginId(TransformPluginType.class, gxdm);
    TransformMeta getXMLDataTransform = new TransformMeta(getXMLDataPid, getXMLDataName, gxdm);
    pipelineMeta.addTransform(getXMLDataTransform);
    GetXmlDataField[] fields = new GetXmlDataField[5];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new GetXmlDataField();
    }
    fields[0].setName("objectid");
    fields[0].setXPath("ObjectID");
    fields[0].setElementType(GetXmlDataField.ELEMENT_TYPE_NODE);
    fields[0].setType(IValueMeta.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(GetXmlDataField.TYPE_TRIM_NONE);
    fields[1].setName("sapident");
    fields[1].setXPath("SAPIDENT");
    fields[1].setElementType(GetXmlDataField.ELEMENT_TYPE_NODE);
    fields[1].setType(IValueMeta.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(GetXmlDataField.TYPE_TRIM_NONE);
    fields[2].setName("quantity");
    fields[2].setXPath("Quantity");
    fields[2].setElementType(GetXmlDataField.ELEMENT_TYPE_NODE);
    fields[2].setType(IValueMeta.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(GetXmlDataField.TYPE_TRIM_NONE);
    fields[3].setName("merkmalname");
    fields[3].setXPath("Merkmalname");
    fields[3].setElementType(GetXmlDataField.ELEMENT_TYPE_NODE);
    fields[3].setType(IValueMeta.TYPE_STRING);
    fields[3].setFormat("");
    fields[3].setLength(-1);
    fields[3].setPrecision(-1);
    fields[3].setCurrencySymbol("");
    fields[3].setDecimalSymbol("");
    fields[3].setGroupSymbol("");
    fields[3].setTrimType(GetXmlDataField.TYPE_TRIM_NONE);
    fields[4].setName("merkmalswert");
    fields[4].setXPath("Merkmalswert");
    fields[4].setElementType(GetXmlDataField.ELEMENT_TYPE_NODE);
    fields[4].setType(IValueMeta.TYPE_STRING);
    fields[4].setFormat("");
    fields[4].setLength(-1);
    fields[4].setPrecision(-1);
    fields[4].setCurrencySymbol("");
    fields[4].setDecimalSymbol("");
    fields[4].setGroupSymbol("");
    fields[4].setTrimType(GetXmlDataField.TYPE_TRIM_NONE);
    gxdm.setEncoding("UTF-8");
    gxdm.setIsAFile(false);
    gxdm.setInFields(true);
    gxdm.setLoopXPath("Level1/Level2/Props");
    gxdm.setXMLField("field1");
    gxdm.setInputFields(fields);
    PipelineHopMeta hi = new PipelineHopMeta(injectorTransform, getXMLDataTransform);
    pipelineMeta.addPipelineHop(hi);
    // 
    // Create a dummy transform 1
    // 
    String dummyTransformName1 = "dummy transform 1";
    DummyMeta dm1 = new DummyMeta();
    String dummyPid1 = registry.getPluginId(TransformPluginType.class, dm1);
    TransformMeta dummyTransform1 = new TransformMeta(dummyPid1, dummyTransformName1, dm1);
    pipelineMeta.addTransform(dummyTransform1);
    PipelineHopMeta hi1 = new PipelineHopMeta(getXMLDataTransform, dummyTransform1);
    pipelineMeta.addPipelineHop(hi1);
    // Now execute the pipeline...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    ITransform si = pipeline.getTransform(dummyTransformName1, 0);
    RowTransformCollector dummyRc1 = new RowTransformCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
    pipeline.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    pipeline.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows);
}

Example 30 with LocalPipelineEngine

use of org.apache.hop.pipeline.engines.local.LocalPipelineEngine in project hop by apache.

the class BaseParsingTest method beforeCommon.

/**
 * Initialize transform info. Method is final against redefine in descendants.
 */
@Before
public final void beforeCommon() throws Exception {
    HopEnvironment.init();
    PluginRegistry.addPluginType(CompressionPluginType.getInstance());
    PluginRegistry.init();
    transformMeta = new TransformMeta();
    transformMeta.setName("test");
    pipeline = new LocalPipelineEngine();
    pipeline.setLogChannel(log);
    pipeline.setRunning(true);
    pipelineMeta = new PipelineMeta() {

        @Override
        public TransformMeta findTransform(String name) {
            return transformMeta;
        }
    };
    fs = VFS.getManager();
    inPrefix = '/' + this.getClass().getPackage().getName().replace('.', '/') + "/files/";
}