Examples with PipelineMeta org.apache.hop.pipeline.PipelineMeta used on opensource projects

Example 96 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class XmlOutputMetaTest method testCheck.

@Test
public void testCheck() throws Exception {
    XmlOutputMeta xmlOutputMeta = new XmlOutputMeta();
    xmlOutputMeta.setDefault();
    PipelineMeta pipelineMeta = mock(PipelineMeta.class);
    TransformMeta transformInfo = mock(TransformMeta.class);
    IRowMeta prev = mock(IRowMeta.class);
    IHopMetadataProvider metadataProvider = mock(IHopMetadataProvider.class);
    IRowMeta info = mock(IRowMeta.class);
    ArrayList<ICheckResult> remarks = new ArrayList<>();
    xmlOutputMeta.check(remarks, pipelineMeta, transformInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), metadataProvider);
    assertEquals(2, remarks.size());
    assertEquals("Transform is receiving info from other transforms.", remarks.get(0).getText());
    assertEquals("File specifications are not checked.", remarks.get(1).getText());
    XmlField xmlField = new XmlField();
    xmlField.setFieldName("aField");
    xmlField.setType(1);
    xmlField.setLength(10);
    xmlField.setPrecision(3);
    xmlOutputMeta.setOutputFields(new XmlField[] { xmlField });
    when(prev.size()).thenReturn(1);
    remarks.clear();
    xmlOutputMeta.check(remarks, pipelineMeta, transformInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), metadataProvider);
    assertEquals(4, remarks.size());
    assertEquals("Transform is connected to previous one, receiving 1 fields", remarks.get(0).getText());
    assertEquals("All output fields are found in the input stream.", remarks.get(1).getText());
    assertEquals("Transform is receiving info from other transforms.", remarks.get(2).getText());
    assertEquals("File specifications are not checked.", remarks.get(3).getText());
}

Example 97 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class XsdValidatorIntTest method testVfsFileTypes.

private void testVfsFileTypes(String dataFilename, String schemaFilename, boolean expected) throws Exception {
    assertNotNull(dataFilename);
    assertNotNull(schemaFilename);
    assertTrue(HopVfs.getFileObject(dataFilename).exists());
    assertTrue(HopVfs.getFileObject(schemaFilename).exists());
    IRowMeta inputRowMeta = new RowMeta();
    inputRowMeta.addValueMeta(new ValueMetaString("DataFile"));
    inputRowMeta.addValueMeta(new ValueMetaString("SchemaFile"));
    List<RowMetaAndData> inputData = new ArrayList<>();
    inputData.add(new RowMetaAndData(inputRowMeta, new Object[] { dataFilename, schemaFilename }));
    String TransformName = "XSD Validator";
    XsdValidatorMeta meta = new XsdValidatorMeta();
    meta.setDefault();
    meta.setXMLSourceFile(true);
    meta.setXMLStream("DataFile");
    meta.setXSDSource(meta.SPECIFY_FIELDNAME);
    meta.setXSDDefinedField("SchemaFile");
    meta.setAddValidationMessage(true);
    PipelineMeta pipelineMeta = PipelineTestFactory.generateTestTransformation(null, meta, TransformName);
    List<RowMetaAndData> result = null;
    result = PipelineTestFactory.executeTestTransformation(pipelineMeta, PipelineTestFactory.INJECTOR_TRANSFORMNAME, TransformName, PipelineTestFactory.DUMMY_TRANSFORMNAME, inputData);
    assertNotNull(result);
    assertEquals(1, result.size());
    // Check Metadata
    assertEquals(IValueMeta.TYPE_STRING, result.get(0).getValueMeta(0).getType());
    assertEquals(IValueMeta.TYPE_STRING, result.get(0).getValueMeta(1).getType());
    assertEquals(IValueMeta.TYPE_BOOLEAN, result.get(0).getValueMeta(2).getType());
    assertEquals("DataFile", result.get(0).getValueMeta(0).getName());
    assertEquals("SchemaFile", result.get(0).getValueMeta(1).getName());
    assertEquals("result", result.get(0).getValueMeta(2).getName());
    // Check result
    assertEquals(dataFilename, result.get(0).getString(0, "default"));
    assertEquals(schemaFilename, result.get(0).getString(1, "default"));
    assertEquals(expected, result.get(0).getBoolean(2, !expected));
}

Example 98 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class XsltTest method runTestWithParams.

public void runTestWithParams(String xmlFieldname, String resultFieldname, boolean xslInField, boolean xslFileInField, String xslFileField, String xslFilename, String xslFactory) throws Exception {
    HopEnvironment.init();
    // 
    // Create a new pipeline...
    // 
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setName("xslt");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector transform...
    // 
    String injectorTransformName = "injector transform";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(TransformPluginType.class, im);
    TransformMeta injectorTransform = new TransformMeta(injectorPid, injectorTransformName, im);
    pipelineMeta.addTransform(injectorTransform);
    // 
    // Create a XSLT transform
    // 
    String xsltName = "xslt transform";
    XsltMeta xm = new XsltMeta();
    String xsltPid = registry.getPluginId(TransformPluginType.class, xm);
    TransformMeta xsltTransform = new TransformMeta(xsltPid, xsltName, xm);
    pipelineMeta.addTransform(xsltTransform);
    TextFileInputField[] fields = new TextFileInputField[3];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new TextFileInputField();
    }
    fields[0].setName("XML");
    fields[0].setType(IValueMeta.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    fields[1].setName("XSL");
    fields[1].setType(IValueMeta.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    fields[2].setName("filename");
    fields[2].setType(IValueMeta.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    xm.setFieldname(xmlFieldname);
    xm.setResultfieldname(resultFieldname);
    xm.setXSLField(xslInField);
    xm.setXSLFileField(xslFileField);
    xm.setXSLFieldIsAFile(xslFileInField);
    xm.setXslFilename(xslFilename);
    xm.setXSLFactory(xslFactory);
    PipelineHopMeta hi = new PipelineHopMeta(injectorTransform, xsltTransform);
    pipelineMeta.addPipelineHop(hi);
    // 
    // Create a dummy transform 1
    // 
    String dummyTransformName1 = "dummy transform 1";
    DummyMeta dm1 = new DummyMeta();
    String dummyPid1 = registry.getPluginId(TransformPluginType.class, dm1);
    TransformMeta dummyTransform1 = new TransformMeta(dummyPid1, dummyTransformName1, dm1);
    pipelineMeta.addTransform(dummyTransform1);
    PipelineHopMeta hi1 = new PipelineHopMeta(xsltTransform, dummyTransform1);
    pipelineMeta.addPipelineHop(hi1);
    // Now execute the pipeline...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    ITransform si = pipeline.getTransform(dummyTransformName1, 0);
    RowTransformCollector dummyRc1 = new RowTransformCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
    pipeline.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(xslFilename);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    pipeline.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows, 2);
}

Example 99 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class HopGuiFileDelegate method exportToSvg.

public void exportToSvg() {
    try {
        String svgXml = null;
        IVariables variables = null;
        String proposedName = null;
        HopGuiPipelineGraph pipelineGraph = HopGui.getActivePipelineGraph();
        if (pipelineGraph != null) {
            PipelineMeta pipelineMeta = pipelineGraph.getPipelineMeta();
            variables = pipelineGraph.getVariables();
            svgXml = PipelineSvgPainter.generatePipelineSvg(pipelineMeta, 1.0f, pipelineGraph.getVariables());
            proposedName = pipelineMeta.getName() + ".svg";
        }
        HopGuiWorkflowGraph workflowGraph = HopGui.getActiveWorkflowGraph();
        if (workflowGraph != null) {
            WorkflowMeta workflowMeta = workflowGraph.getWorkflowMeta();
            variables = workflowGraph.getVariables();
            svgXml = WorkflowSvgPainter.generateWorkflowSvg(workflowMeta, 1.0f, workflowGraph.getVariables());
            proposedName = workflowMeta.getName() + ".svg";
        }
        if (svgXml != null) {
            String proposedFilename = variables.getVariable("user.home") + File.separator + proposedName;
            FileObject proposedFile = HopVfs.getFileObject(proposedFilename);
            String filename = BaseDialog.presentFileDialog(true, hopGui.getShell(), null, variables, proposedFile, new String[] { "*.svg" }, new String[] { "SVG Files" }, true);
            if (filename != null) {
                String realFilename = variables.resolve(filename);
                FileObject file = HopVfs.getFileObject(realFilename);
                if (file.exists()) {
                    MessageBox box = new MessageBox(hopGui.getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);
                    box.setText("File exists");
                    box.setMessage("This file already exists. Do you want to overwrite it?");
                    int answer = box.open();
                    if ((answer & SWT.YES) == 0) {
                        return;
                    }
                }
                OutputStream outputStream = null;
                try {
                    outputStream = HopVfs.getOutputStream(file, false);
                    outputStream.write(svgXml.getBytes(Const.XML_ENCODING));
                } finally {
                    if (outputStream != null) {
                        outputStream.close();
                    }
                }
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), "Error", "Error exporting to SVG", e);
    }
}

Example 100 with PipelineMeta

use of org.apache.hop.pipeline.PipelineMeta in project hop by apache.

the class UpdateMetaTest method setUp.

@Before
public void setUp() throws HopException {
    HopEnvironment.init();
    PluginRegistry.init();
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setName("delete1");
    Map<String, String> vars = new HashMap<>();
    vars.put("max.sz", "10");
    umi = new UpdateMeta();
    ud = new UpdateData();
    PluginRegistry plugReg = PluginRegistry.getInstance();
    String deletePid = plugReg.getPluginId(TransformPluginType.class, umi);
    transformMeta = new TransformMeta(deletePid, "delete", umi);
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.setVariables(vars);
    pipelineMeta.addTransform(transformMeta);
    mockHelper = new TransformMockHelper<>("Update", UpdateMeta.class, UpdateData.class);
    Mockito.when(mockHelper.logChannelFactory.create(Mockito.any(), Mockito.any(ILoggingObject.class))).thenReturn(mockHelper.iLogChannel);
    upd = new Update(transformMeta, umi, ud, 1, pipelineMeta, pipeline);
    List<String> attributes = Arrays.asList("connection", "lookup", "commit", "error_ignored", "ignore_flag_field", "skip_lookup", "use_batch");
    Map<String, String> getterMap = new HashMap<String, String>() {

        {
            put("connection", "getConnection");
            put("lookup", "getLookupField");
            put("commit", "getCommitSize");
            put("error_ignored", "isErrorIgnored");
            put("ignore_flag_field", "getIgnoreFlagField");
            put("skip_lookup", "isSkipLookup");
            put("use_batch", "isUseBatchUpdate");
        }
    };
    Map<String, String> setterMap = new HashMap<String, String>() {

        {
            put("connection", "setConnection");
            put("lookup", "setLookupField");
            put("commit", "setCommitSize");
            put("error_ignored", "setErrorIgnored");
            put("ignore_flag_field", "setIgnoreFlagField");
            put("skip_lookup", "setSkipLookup");
            put("use_batch", "setUseBatchUpdate");
        }
    };
    Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
    Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
    loadSaveTester = new LoadSaveTester(testMetaClass, attributes, getterMap, setterMap, attrValidatorMap, typeValidatorMap, this);
    IFieldLoadSaveValidatorFactory validatorFactory = loadSaveTester.getFieldLoadSaveValidatorFactory();
    validatorFactory.registerValidator(validatorFactory.getName(UpdateLookupField.class), new ObjectValidator<UpdateLookupField>(validatorFactory, UpdateLookupField.class, Arrays.asList("schema", "table", "key", "value"), new HashMap<String, String>() {

        {
            put("schema", "getSchemaName");
            put("table", "getTableName");
            put("key", "getLookupKeys");
            put("value", "getUpdateFields");
        }
    }, new HashMap<String, String>() {

        {
            put("schema", "setSchemaName");
            put("table", "setTableName");
            put("key", "setLookupKeys");
            put("value", "setUpdateFields");
        }
    }));
    validatorFactory.registerValidator(validatorFactory.getName(List.class, UpdateLookupField.class), new ListLoadSaveValidator<UpdateLookupField>(new UpdateLookupFieldLoadSaveValidator()));
    validatorFactory.registerValidator(validatorFactory.getName(UpdateKeyField.class), new ObjectValidator<UpdateKeyField>(validatorFactory, UpdateKeyField.class, Arrays.asList("name", "field", "condition", "name2"), new HashMap<String, String>() {

        {
            put("name", "getKeyStream");
            put("field", "getKeyLookup");
            put("condition", "getKeyCondition");
            put("name2", "getKeyStream2");
        }
    }, new HashMap<String, String>() {

        {
            put("name", "setKeyStream");
            put("field", "setKeyLookup");
            put("condition", "setKeyCondition");
            put("name2", "setKeyStream2");
        }
    }));
    validatorFactory.registerValidator(validatorFactory.getName(List.class, UpdateKeyField.class), new ListLoadSaveValidator<UpdateKeyField>(new UpdateKeyFieldLoadSaveValidator()));
    validatorFactory.registerValidator(validatorFactory.getName(UpdateField.class), new ObjectValidator<UpdateField>(validatorFactory, UpdateField.class, Arrays.asList("name", "rename"), new HashMap<String, String>() {

        {
            put("name", "getUpdateLookup");
            put("rename", "getUpdateStream");
        }
    }, new HashMap<String, String>() {

        {
            put("name", "setUpdateLookup");
            put("rename", "setUpdateStream");
        }
    }));
    validatorFactory.registerValidator(validatorFactory.getName(List.class, UpdateField.class), new ListLoadSaveValidator<UpdateField>(new UpdateFieldLoadSaveValidator()));
}