Examples with TransMeta org.pentaho.di.trans.TransMeta used on opensource projects

Example 71 with TransMeta

use of org.pentaho.di.trans.TransMeta in project pentaho-kettle by pentaho.

the class RepositoryImporter method importAll.

@Override
public synchronized void importAll(RepositoryImportFeedbackInterface feedback, String fileDirectory, String[] filenames, RepositoryDirectoryInterface baseDirectory, boolean overwrite, boolean continueOnError, String versionComment) {
    this.baseDirectory = baseDirectory;
    this.overwrite = overwrite;
    this.continueOnError = continueOnError;
    this.versionComment = versionComment;
    String importPathCompatibility = System.getProperty(Const.KETTLE_COMPATIBILITY_IMPORT_PATH_ADDITION_ON_VARIABLES, "N");
    this.needToCheckPathForVariables = "N".equalsIgnoreCase(importPathCompatibility);
    askReplace = Props.getInstance().askAboutReplacingDatabaseConnections();
    if (askReplace) {
        if (feedback instanceof HasOverwritePrompter) {
            Props.getInstance().setProperty(IMPORT_ASK_ABOUT_REPLACE_CS, "Y");
            Props.getInstance().setProperty(IMPORT_ASK_ABOUT_REPLACE_DB, "Y");
            Props.getInstance().setProperty(IMPORT_ASK_ABOUT_REPLACE_PS, "Y");
            Props.getInstance().setProperty(IMPORT_ASK_ABOUT_REPLACE_SS, "Y");
            this.overwritePrompter = ((HasOverwritePrompter) feedback).getOverwritePrompter();
        } else {
            this.overwritePrompter = new OverwritePrompter() {

                @Override
                public boolean overwritePrompt(String arg0, String arg1, String arg2) {
                    throw new RuntimeException(BaseMessages.getString(PKG, "RepositoryImporter.CannotPrompt.Label"));
                }
            };
        }
    } else {
        final boolean replaceExisting = Props.getInstance().replaceExistingDatabaseConnections();
        this.overwritePrompter = new OverwritePrompter() {

            @Override
            public boolean overwritePrompt(String arg0, String arg1, String arg2) {
                return replaceExisting;
            }
        };
    }
    referencingObjects = new ArrayList<RepositoryObject>();
    feedback.setLabel(BaseMessages.getString(PKG, "RepositoryImporter.ImportXML.Label"));
    try {
        loadSharedObjects();
        RepositoryImportLocation.setRepositoryImportLocation(baseDirectory);
        for (int ii = 0; ii < filenames.length; ++ii) {
            final String filename = (!Utils.isEmpty(fileDirectory)) ? fileDirectory + Const.FILE_SEPARATOR + filenames[ii] : filenames[ii];
            if (log.isBasic()) {
                log.logBasic("Import objects from XML file [" + filename + "]");
            }
            feedback.addLog(BaseMessages.getString(PKG, "RepositoryImporter.WhichFile.Log", filename));
            // To where?
            feedback.setLabel(BaseMessages.getString(PKG, "RepositoryImporter.WhichDir.Label"));
            // 
            try {
                RepositoryExportSaxParser parser = new RepositoryExportSaxParser(filename, feedback);
                parser.parse(this);
            } catch (Exception e) {
                addException(e);
                feedback.showError(BaseMessages.getString(PKG, "RepositoryImporter.ErrorGeneral.Title"), BaseMessages.getString(PKG, "RepositoryImporter.ErrorGeneral.Message"), e);
            }
        }
        // Correct those jobs and transformations that contain references to other objects.
        for (RepositoryObject repoObject : referencingObjects) {
            switch(repoObject.getObjectType()) {
                case TRANSFORMATION:
                    TransMeta transMeta = rep.loadTransformation(repoObject.getObjectId(), null);
                    saveTransformationToRepo(transMeta, feedback);
                    break;
                case JOB:
                    JobMeta jobMeta = rep.loadJob(repoObject.getObjectId(), null);
                    saveJobToRepo(jobMeta, feedback);
                    break;
                default:
                    throw new KettleException(BaseMessages.getString(PKG, "RepositoryImporter.ErrorDetectFileType"));
            }
        }
        feedback.addLog(BaseMessages.getString(PKG, "RepositoryImporter.ImportFinished.Log"));
    } catch (Exception e) {
        addException(e);
        feedback.showError(BaseMessages.getString(PKG, "RepositoryImporter.ErrorGeneral.Title"), BaseMessages.getString(PKG, "RepositoryImporter.ErrorGeneral.Message"), e);
    } finally {
        // set the repository import location to null when done!
        RepositoryImportLocation.setRepositoryImportLocation(null);
    }
}

Example 72 with TransMeta

use of org.pentaho.di.trans.TransMeta in project pentaho-kettle by pentaho.

the class KettleFileRepository method save.

public void save(RepositoryElementInterface repositoryElement, String versionComment, ProgressMonitorListener monitor, ObjectId parentId, boolean used) throws KettleException {
    try {
        if (!(repositoryElement instanceof XMLInterface) && !(repositoryElement instanceof SharedObjectInterface)) {
            throw new KettleException("Class [" + repositoryElement.getClass().getName() + "] needs to implement the XML Interface in order to save it to disk");
        }
        if (!Utils.isEmpty(versionComment)) {
            insertLogEntry("Save repository element : " + repositoryElement.toString() + " : " + versionComment);
        }
        ObjectId objectId = new StringObjectId(calcObjectId(repositoryElement));
        FileObject fileObject = getFileObject(repositoryElement);
        String xml = ((XMLInterface) repositoryElement).getXML();
        OutputStream os = KettleVFS.getOutputStream(fileObject, false);
        os.write(xml.getBytes(Const.XML_ENCODING));
        os.close();
        if (repositoryElement instanceof ChangedFlagInterface) {
            ((ChangedFlagInterface) repositoryElement).clearChanged();
        }
        // 
        if (repositoryElement.getObjectId() != null && !repositoryElement.getObjectId().equals(objectId)) {
            delObject(repositoryElement.getObjectId());
        }
        repositoryElement.setObjectId(objectId);
        // 
        if (repositoryElement instanceof TransMeta) {
            ((TransMeta) repositoryElement).saveMetaStoreObjects(this, metaStore);
        }
        if (repositoryElement instanceof JobMeta) {
            ((JobMeta) repositoryElement).saveMetaStoreObjects(this, metaStore);
        }
    } catch (Exception e) {
        throw new KettleException("Unable to save repository element [" + repositoryElement + "] to XML file : " + calcFilename(repositoryElement), e);
    }
}

Example 73 with TransMeta

use of org.pentaho.di.trans.TransMeta in project pentaho-kettle by pentaho.

the class GetXMLDataTest method testInit.

public void testInit() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("getxmldata1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a Get XML Data step
    // 
    String getXMLDataName = "get xml data step";
    GetXMLDataMeta gxdm = new GetXMLDataMeta();
    String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
    StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
    transMeta.addStep(getXMLDataStep);
    GetXMLDataField[] fields = new GetXMLDataField[5];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new GetXMLDataField();
    }
    fields[0].setName("objectid");
    fields[0].setXPath("${xml_path}");
    fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    gxdm.setEncoding("UTF-8");
    gxdm.setIsAFile(false);
    gxdm.setInFields(true);
    gxdm.setLoopXPath("Level1/Level2/Props");
    gxdm.setXMLField("field1");
    gxdm.setInputFields(fields);
    TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    GetXMLDataData getXMLDataData = new GetXMLDataData();
    GetXMLData getXmlData = new GetXMLData(dummyStep1, getXMLDataData, 0, transMeta, trans);
    getXmlData.setVariable("xml_path", "data/owner");
    getXmlData.init(gxdm, getXMLDataData);
    assertEquals(gxdm.getInputFields()[0].getXPath(), "${xml_path}");
}

Example 74 with TransMeta

use of org.pentaho.di.trans.TransMeta in project pentaho-kettle by pentaho.

the class XMLOutputMetaTest method testCheck.

@Test
public void testCheck() throws Exception {
    XMLOutputMeta xmlOutputMeta = new XMLOutputMeta();
    xmlOutputMeta.setDefault();
    TransMeta transMeta = mock(TransMeta.class);
    StepMeta stepInfo = mock(StepMeta.class);
    RowMetaInterface prev = mock(RowMetaInterface.class);
    Repository repos = mock(Repository.class);
    IMetaStore metastore = mock(IMetaStore.class);
    RowMetaInterface info = mock(RowMetaInterface.class);
    ArrayList<CheckResultInterface> remarks = new ArrayList<>();
    xmlOutputMeta.check(remarks, transMeta, stepInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), repos, metastore);
    assertEquals(2, remarks.size());
    assertEquals("Step is receiving info from other steps.", remarks.get(0).getText());
    assertEquals("File specifications are not checked.", remarks.get(1).getText());
    XMLField xmlField = new XMLField();
    xmlField.setFieldName("aField");
    xmlField.setType(1);
    xmlField.setLength(10);
    xmlField.setPrecision(3);
    xmlOutputMeta.setOutputFields(new XMLField[] { xmlField });
    when(prev.size()).thenReturn(1);
    remarks.clear();
    xmlOutputMeta.check(remarks, transMeta, stepInfo, prev, new String[] { "input" }, new String[] { "output" }, info, new Variables(), repos, metastore);
    assertEquals(4, remarks.size());
    assertEquals("Step is connected to previous one, receiving 1 fields", remarks.get(0).getText());
    assertEquals("All output fields are found in the input stream.", remarks.get(1).getText());
    assertEquals("Step is receiving info from other steps.", remarks.get(2).getText());
    assertEquals("File specifications are not checked.", remarks.get(3).getText());
}

Example 75 with TransMeta

use of org.pentaho.di.trans.TransMeta in project pentaho-kettle by pentaho.

the class XsltTest method runTestWithParams.

public void runTestWithParams(String xmlFieldname, String resultFieldname, boolean xslInField, boolean xslFileInField, String xslFileField, String xslFilename, String xslFactory) throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("xslt");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a XSLT step
    // 
    String xsltName = "xslt step";
    XsltMeta xm = new XsltMeta();
    String xsltPid = registry.getPluginId(StepPluginType.class, xm);
    StepMeta xsltStep = new StepMeta(xsltPid, xsltName, xm);
    transMeta.addStep(xsltStep);
    TextFileInputField[] fields = new TextFileInputField[3];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new TextFileInputField();
    }
    fields[0].setName("XML");
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    fields[1].setName("XSL");
    fields[1].setType(ValueMetaInterface.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    fields[2].setName("filename");
    fields[2].setType(ValueMetaInterface.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    xm.setFieldname(xmlFieldname);
    xm.setResultfieldname(resultFieldname);
    xm.setXSLField(xslInField);
    xm.setXSLFileField(xslFileField);
    xm.setXSLFieldIsAFile(xslFileInField);
    xm.setXslFilename(xslFilename);
    xm.setXSLFactory(xslFactory);
    TransHopMeta hi = new TransHopMeta(injectorStep, xsltStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(xsltStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(xslFilename);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows, 2);
}