Examples with PluginRegistry org.pentaho.di.core.plugins.PluginRegistry used on opensource projects

Example 41 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class WebServer method startServer.

public void startServer() throws Exception {
    server = new Server();
    List<String> roles = new ArrayList<>();
    roles.add(Constraint.ANY_ROLE);
    // Set up the security handler, optionally with JAAS
    // 
    ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler();
    if (System.getProperty("loginmodulename") != null && System.getProperty("java.security.auth.login.config") != null) {
        JAASLoginService jaasLoginService = new JAASLoginService(SERVICE_NAME);
        jaasLoginService.setLoginModuleName(System.getProperty("loginmodulename"));
        securityHandler.setLoginService(jaasLoginService);
    } else {
        roles.add(DEFAULT_ROLE);
        HashLoginService hashLoginService;
        SlaveServer slaveServer = transformationMap.getSlaveServerConfig().getSlaveServer();
        if (!Utils.isEmpty(slaveServer.getPassword())) {
            hashLoginService = new HashLoginService(SERVICE_NAME);
            UserStore userStore = new UserStore();
            userStore.addUser(slaveServer.getUsername(), new Password(slaveServer.getPassword()), new String[] { DEFAULT_ROLE });
            hashLoginService.setUserStore(userStore);
        } else {
            // See if there is a kettle.pwd file in the KETTLE_HOME directory:
            if (Utils.isEmpty(passwordFile)) {
                File homePwdFile = new File(Const.getKettleCartePasswordFile());
                if (homePwdFile.exists()) {
                    passwordFile = Const.getKettleCartePasswordFile();
                } else {
                    passwordFile = Const.getKettleLocalCartePasswordFile();
                }
            }
            hashLoginService = new HashLoginService(SERVICE_NAME, passwordFile) {

                @Override
                protected String[] loadRoleInfo(UserPrincipal user) {
                    List<String> newRoles = new ArrayList<>();
                    newRoles.add(DEFAULT_ROLE);
                    String[] roles = super.loadRoleInfo(user);
                    if (null != roles) {
                        Collections.addAll(newRoles, roles);
                    }
                    return newRoles.toArray(new String[0]);
                }
            };
        }
        securityHandler.setLoginService(hashLoginService);
    }
    Constraint constraint = new Constraint();
    constraint.setName(Constraint.__BASIC_AUTH);
    constraint.setRoles(roles.toArray(new String[0]));
    constraint.setAuthenticate(true);
    ConstraintMapping constraintMapping = new ConstraintMapping();
    constraintMapping.setConstraint(constraint);
    constraintMapping.setPathSpec("/*");
    securityHandler.setConstraintMappings(new ConstraintMapping[] { constraintMapping });
    // Add all the servlets defined in kettle-servlets.xml ...
    // 
    ContextHandlerCollection contexts = new ContextHandlerCollection();
    // Root
    // 
    ServletContextHandler root = new ServletContextHandler(contexts, GetRootServlet.CONTEXT_PATH, ServletContextHandler.SESSIONS);
    GetRootServlet rootServlet = new GetRootServlet();
    rootServlet.setJettyMode(true);
    root.addServlet(new ServletHolder(rootServlet), "/*");
    PluginRegistry pluginRegistry = PluginRegistry.getInstance();
    List<PluginInterface> plugins = pluginRegistry.getPlugins(CartePluginType.class);
    for (PluginInterface plugin : plugins) {
        CartePluginInterface servlet = pluginRegistry.loadClass(plugin, CartePluginInterface.class);
        servlet.setup(transformationMap, jobMap, socketRepository, detections);
        servlet.setJettyMode(true);
        ServletContextHandler servletContext = new ServletContextHandler(contexts, getContextPath(servlet), ServletContextHandler.SESSIONS);
        ServletHolder servletHolder = new ServletHolder((Servlet) servlet);
        servletContext.addServlet(servletHolder, "/*");
    }
    // setup jersey (REST)
    ServletHolder jerseyServletHolder = new ServletHolder(ServletContainer.class);
    jerseyServletHolder.setInitParameter("com.sun.jersey.config.property.resourceConfigClass", "com.sun.jersey.api.core.PackagesResourceConfig");
    jerseyServletHolder.setInitParameter("com.sun.jersey.config.property.packages", "org.pentaho.di.www.jaxrs");
    root.addServlet(jerseyServletHolder, "/api/*");
    // setup static resource serving
    // ResourceHandler mobileResourceHandler = new ResourceHandler();
    // mobileResourceHandler.setWelcomeFiles(new String[]{"index.html"});
    // mobileResourceHandler.setResourceBase(getClass().getClassLoader().
    // getResource("org/pentaho/di/www/mobile").toExternalForm());
    // Context mobileContext = new Context(contexts, "/mobile", Context.SESSIONS);
    // mobileContext.setHandler(mobileResourceHandler);
    // Allow png files to be shown for transformations and jobs...
    // 
    ResourceHandler resourceHandler = new ResourceHandler();
    resourceHandler.setResourceBase("temp");
    // add all handlers/contexts to server
    // set up static servlet
    ServletHolder staticHolder = new ServletHolder("static", DefaultServlet.class);
    // resourceBase maps to the path relative to where carte is started
    staticHolder.setInitParameter("resourceBase", "./static/");
    staticHolder.setInitParameter("dirAllowed", "true");
    staticHolder.setInitParameter("pathInfoOnly", "true");
    root.addServlet(staticHolder, "/static/*");
    HandlerList handlers = new HandlerList();
    handlers.setHandlers(new Handler[] { resourceHandler, contexts });
    securityHandler.setHandler(handlers);
    server.setHandler(securityHandler);
    // Start execution
    createListeners();
    server.start();
}

Example 42 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class MySQLBulkLoaderTest method testEscapeCharacters.

@Test
public void testEscapeCharacters() throws KettleException, IOException {
    PluginRegistry.addPluginType(ValueMetaPluginType.getInstance());
    PluginRegistry.init(false);
    MySQLBulkLoader loader;
    MySQLBulkLoaderData ld = new MySQLBulkLoaderData();
    MySQLBulkLoaderMeta lm = new MySQLBulkLoaderMeta();
    TransMeta transMeta = new TransMeta();
    transMeta.setName("loader");
    PluginRegistry plugReg = PluginRegistry.getInstance();
    String loaderPid = plugReg.getPluginId(StepPluginType.class, lm);
    StepMeta stepMeta = new StepMeta(loaderPid, "loader", lm);
    Trans trans = new Trans(transMeta);
    transMeta.addStep(stepMeta);
    trans.setRunning(true);
    loader = spy(new MySQLBulkLoader(stepMeta, ld, 1, transMeta, trans));
    RowMeta rm = new RowMeta();
    ValueMetaString vm = new ValueMetaString("I don't want NPE!");
    rm.addValueMeta(vm);
    RowMeta spyRowMeta = spy(new RowMeta());
    when(spyRowMeta.getValueMeta(anyInt())).thenReturn(vm);
    loader.setInputRowMeta(spyRowMeta);
    MySQLBulkLoaderMeta smi = new MySQLBulkLoaderMeta();
    smi.setFieldStream(new String[] { "Test" });
    smi.setFieldFormatType(new int[] { MySQLBulkLoaderMeta.FIELD_FORMAT_TYPE_STRING_ESCAPE });
    smi.setEscapeChar("\\");
    smi.setEnclosure("\"");
    smi.setDatabaseMeta(mock(DatabaseMeta.class));
    MySQLBulkLoaderData sdi = new MySQLBulkLoaderData();
    sdi.keynrs = new int[1];
    sdi.keynrs[0] = 0;
    sdi.fifoStream = mock(OutputStream.class);
    loader.init(smi, sdi);
    loader.first = false;
    when(loader.getRow()).thenReturn(new String[] { "test\"Escape\\" });
    loader.processRow(smi, sdi);
    verify(sdi.fifoStream, times(1)).write("test\\\"Escape\\\\".getBytes());
}

Example 43 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class MappingIT method testInfoStreams_with_main_data_path.

/**
 * Tests that an input step that is a main data path is not flagged as an info stream
 */
public void testInfoStreams_with_main_data_path() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Mapping Info Test");
    StepMeta rowGenerator = buildRowGeneratorStep(registry, "Generate Rows");
    transMeta.addStep(rowGenerator);
    StepMeta rowGeneratorMain = buildRowGeneratorStep(registry, "Generate Rows Main");
    transMeta.addStep(rowGeneratorMain);
    String mappingName = "mapping";
    MappingMeta mappingMeta = new MappingMeta();
    mappingMeta.setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
    mappingMeta.setFileName("test/org/pentaho/di/trans/steps/mapping/subtrans.ktr");
    List<MappingIODefinition> inputMappings = new ArrayList<MappingIODefinition>();
    String mappingInputStepName = "input";
    inputMappings.add(createMappingDef(rowGenerator.getName(), mappingInputStepName, "string", "a"));
    // Create the main data path mapping
    MappingIODefinition mainMappingDef = createMappingDef(rowGeneratorMain.getName(), mappingInputStepName, "string", "a");
    mainMappingDef.setMainDataPath(true);
    inputMappings.add(mainMappingDef);
    mappingMeta.setInputMappings(inputMappings);
    String mappingPid = registry.getPluginId(StepPluginType.class, mappingMeta);
    StepMeta mapping = new StepMeta(mappingPid, mappingName, mappingMeta);
    transMeta.addStep(mapping);
    TransHopMeta hopGeneratorToMapping = new TransHopMeta(rowGenerator, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    hopGeneratorToMapping = new TransHopMeta(rowGeneratorMain, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // (Copied from TransMeta.loadXML())
    for (int i = 0; i < transMeta.nrSteps(); i++) {
        StepMeta stepMeta = transMeta.getStep(i);
        StepMetaInterface sii = stepMeta.getStepMetaInterface();
        if (sii != null) {
            sii.searchInfoAndTargetSteps(transMeta.getSteps());
        }
    }
    // Verify the transformation was configured properly
    assertEquals("Transformation not initialized properly", 3, transMeta.nrSteps());
    StepMeta meta = transMeta.getStep(2);
    assertTrue("Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta);
    MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
    assertEquals("Expected a two input mapping definition", 2, loadedMappingMeta.getInputMappings().size());
    StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
    assertEquals("Expected a single Info Stream", 1, ioMeta.getInfoStreams().size());
    assertEquals("Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length);
}

Example 44 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class MappingIT method testInfoStreams_single.

/**
 * Tests that info steps are correctly identified via StepMetaInterface.getStepIOMeta()
 */
public void testInfoStreams_single() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Mapping Info Test");
    StepMeta rowGenerator = buildRowGeneratorStep(registry, "Generate Rows");
    transMeta.addStep(rowGenerator);
    String mappingName = "mapping";
    MappingMeta mappingMeta = new MappingMeta();
    mappingMeta.setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
    mappingMeta.setFileName("test/org/pentaho/di/trans/steps/mapping/subtrans.ktr");
    String mappingInputStepName = "input";
    mappingMeta.setInputMappings(Collections.singletonList(createMappingDef(rowGenerator.getName(), mappingInputStepName, "string", "a")));
    String mappingPid = registry.getPluginId(StepPluginType.class, mappingMeta);
    StepMeta mapping = new StepMeta(mappingPid, mappingName, mappingMeta);
    transMeta.addStep(mapping);
    TransHopMeta hopGeneratorToMapping = new TransHopMeta(rowGenerator, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // (Copied from TransMeta.loadXML())
    for (int i = 0; i < transMeta.nrSteps(); i++) {
        StepMeta stepMeta = transMeta.getStep(i);
        StepMetaInterface sii = stepMeta.getStepMetaInterface();
        if (sii != null) {
            sii.searchInfoAndTargetSteps(transMeta.getSteps());
        }
    }
    // Verify the transformation was configured properly
    assertEquals("Transformation not initialized properly", 2, transMeta.nrSteps());
    StepMeta meta = transMeta.getStep(1);
    assertTrue("Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta);
    MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
    assertEquals("Expected a single input mapping definition", 1, loadedMappingMeta.getInputMappings().size());
    StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
    assertEquals("Expected a single Info Stream", 1, ioMeta.getInfoStreams().size());
    assertEquals("Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length);
    // Verify the transformation can be executed
    StepInterface si = trans.getStepInterface(mappingName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    trans.startThreads();
    trans.waitUntilFinished();
    assertEquals(1, rc.getRowsRead().size());
    assertEquals(1, rc.getRowsWritten().size());
}

Example 45 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class InsertUpdateIT method setUp.

@Override
@Before
public void setUp() throws Exception {
    KettleEnvironment.init();
    /* SET UP TRANSFORMATION */
    // Create a new transformation...
    TransMeta transMeta = new TransMeta();
    transMeta.setName("insert/update test");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    /* SET UP DATABASE */
    // Create target table
    db = new Database(transMeta, dbInfo);
    db.connect();
    String source = db.getCreateTableStatement(TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true);
    db.execStatement(source);
    // populate target table
    for (String sql : insertStatement) {
        db.execStatement(sql);
    }
    /* SET UP TRANSFORMATION STEPS */
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step...
    String injectorStepName = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im);
    transMeta.addStep(injectorStep);
    // create the update step...
    String updateStepName = "insert/update [" + TARGET_TABLE + "]";
    insupd = new InsertUpdateMeta();
    insupd.setDatabaseMeta(transMeta.findDatabase("db"));
    insupd.setTableName(TARGET_TABLE);
    insupd.setUpdateLookup(new String[] { "VALUE", "ROW_ORDER" });
    insupd.setUpdateStream(new String[] { "VALUE", "ROW_ORDER" });
    insupd.setUpdate(new Boolean[] { true, false });
    String fromid = registry.getPluginId(StepPluginType.class, insupd);
    StepMeta updateStep = new StepMeta(fromid, updateStepName, insupd);
    updateStep.setDescription("insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]");
    transMeta.addStep(updateStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, updateStep);
    transMeta.addTransHop(hi);
    /* PREPARE TRANSFORMATION EXECUTION */
    trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(updateStepName, 0);
    rc = new RowStepCollector();
    si.addRowListener(rc);
    rp = trans.addRowProducer(injectorStepName, 0);
}