Examples with InterpreterSetting org.apache.zeppelin.interpreter.InterpreterSetting used on opensource projects

Example 16 with InterpreterSetting

use of org.apache.zeppelin.interpreter.InterpreterSetting in project zeppelin by apache.

the class AbstractTestRestApi method start.

private static void start(boolean withAuth) throws Exception {
    if (!wasRunning) {
        System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../");
        System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_WAR.getVarName(), "../zeppelin-web/dist");
        // some test profile does not build zeppelin-web.
        // to prevent zeppelin starting up fail, create zeppelin-web/dist directory
        new File("../zeppelin-web/dist").mkdirs();
        LOG.info("Staring test Zeppelin up...");
        ZeppelinConfiguration conf = ZeppelinConfiguration.create();
        if (withAuth) {
            isRunningWithAuth = true;
            // Set Anonymous session to false.
            System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName(), "false");
            // Create a shiro env test.
            shiroIni = new File("../conf/shiro.ini");
            if (!shiroIni.exists()) {
                shiroIni.createNewFile();
            }
            FileUtils.writeStringToFile(shiroIni, zeppelinShiro);
        }
        // exclude org.apache.zeppelin.rinterpreter.* for scala 2.11 test
        String interpreters = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS);
        String interpretersCompatibleWithScala211Test = null;
        for (String intp : interpreters.split(",")) {
            if (intp.startsWith("org.apache.zeppelin.rinterpreter")) {
                continue;
            }
            if (interpretersCompatibleWithScala211Test == null) {
                interpretersCompatibleWithScala211Test = intp;
            } else {
                interpretersCompatibleWithScala211Test += "," + intp;
            }
        }
        System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), interpretersCompatibleWithScala211Test);
        executor = Executors.newSingleThreadExecutor();
        executor.submit(server);
        long s = System.currentTimeMillis();
        boolean started = false;
        while (System.currentTimeMillis() - s < 1000 * 60 * 3) {
            // 3 minutes
            Thread.sleep(2000);
            started = checkIfServerIsRunning();
            if (started == true) {
                break;
            }
        }
        if (started == false) {
            throw new RuntimeException("Can not start Zeppelin server");
        }
        LOG.info("Test Zeppelin stared.");
        // assume first one is spark
        InterpreterSetting sparkIntpSetting = null;
        for (InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterSettingManager().get()) {
            if (intpSetting.getName().equals("spark")) {
                sparkIntpSetting = intpSetting;
            }
        }
        Properties sparkProperties = (Properties) sparkIntpSetting.getProperties();
        // so configure zeppelin use spark cluster
        if ("true".equals(System.getenv("CI"))) {
            // set spark master and other properties
            sparkProperties.setProperty("master", "local[2]");
            sparkProperties.setProperty("spark.cores.max", "2");
            sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
            // set spark home for pyspark
            sparkProperties.setProperty("spark.home", getSparkHome());
            sparkIntpSetting.setProperties(sparkProperties);
            pySpark = true;
            sparkR = true;
            ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId());
        } else {
            String sparkHome = getSparkHome();
            if (sparkHome != null) {
                if (System.getenv("SPARK_MASTER") != null) {
                    sparkProperties.setProperty("master", System.getenv("SPARK_MASTER"));
                } else {
                    sparkProperties.setProperty("master", "local[2]");
                }
                sparkProperties.setProperty("spark.cores.max", "2");
                // set spark home for pyspark
                sparkProperties.setProperty("spark.home", sparkHome);
                sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
                pySpark = true;
                sparkR = true;
            }
            ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId());
        }
    }
}

Example 17 with InterpreterSetting

use of org.apache.zeppelin.interpreter.InterpreterSetting in project zeppelin by apache.

the class NotebookServer method angularObjectUpdated.

/**
   * When angular object updated from client
   *
   * @param conn the web socket.
   * @param notebook the notebook.
   * @param fromMessage the message.
   */
private void angularObjectUpdated(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage) {
    String noteId = (String) fromMessage.get("noteId");
    String paragraphId = (String) fromMessage.get("paragraphId");
    String interpreterGroupId = (String) fromMessage.get("interpreterGroupId");
    String varName = (String) fromMessage.get("name");
    Object varValue = fromMessage.get("value");
    String user = fromMessage.principal;
    AngularObject ao = null;
    boolean global = false;
    // propagate change to (Remote) AngularObjectRegistry
    Note note = notebook.getNote(noteId);
    if (note != null) {
        List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
        for (InterpreterSetting setting : settings) {
            if (setting.getInterpreterGroup(user, note.getId()) == null) {
                continue;
            }
            if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
                AngularObjectRegistry angularObjectRegistry = setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
                // first trying to get local registry
                ao = angularObjectRegistry.get(varName, noteId, paragraphId);
                if (ao == null) {
                    // then try notebook scope registry
                    ao = angularObjectRegistry.get(varName, noteId, null);
                    if (ao == null) {
                        // then try global scope registry
                        ao = angularObjectRegistry.get(varName, null, null);
                        if (ao == null) {
                            LOG.warn("Object {} is not binded", varName);
                        } else {
                            // path from client -> server
                            ao.set(varValue, false);
                            global = true;
                        }
                    } else {
                        // path from client -> server
                        ao.set(varValue, false);
                        global = false;
                    }
                } else {
                    ao.set(varValue, false);
                    global = false;
                }
                break;
            }
        }
    }
    if (global) {
        // interpreter.
        for (Note n : notebook.getAllNotes()) {
            List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
            for (InterpreterSetting setting : settings) {
                if (setting.getInterpreterGroup(user, n.getId()) == null) {
                    continue;
                }
                if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
                    AngularObjectRegistry angularObjectRegistry = setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
                    this.broadcastExcept(n.getId(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao).put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId()).put("paragraphId", ao.getParagraphId()), conn);
                }
            }
        }
    } else {
        // broadcast to all web session for the note
        this.broadcastExcept(note.getId(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao).put("interpreterGroupId", interpreterGroupId).put("noteId", note.getId()).put("paragraphId", ao.getParagraphId()), conn);
    }
}

Example 18 with InterpreterSetting

use of org.apache.zeppelin.interpreter.InterpreterSetting in project zeppelin by apache.

the class InterpreterBindingUtils method getInterpreterBindings.

public static List<InterpreterSettingsList> getInterpreterBindings(Notebook notebook, String noteId) {
    List<InterpreterSettingsList> settingList = new LinkedList<>();
    List<InterpreterSetting> selectedSettings = notebook.getBindedInterpreterSettings(noteId);
    for (InterpreterSetting setting : selectedSettings) {
        settingList.add(new InterpreterSettingsList(setting.getId(), setting.getName(), setting.getInterpreterInfos(), true));
    }
    List<InterpreterSetting> availableSettings = notebook.getInterpreterSettingManager().get();
    for (InterpreterSetting setting : availableSettings) {
        boolean selected = false;
        for (InterpreterSetting selectedSetting : selectedSettings) {
            if (selectedSetting.getId().equals(setting.getId())) {
                selected = true;
                break;
            }
        }
        if (!selected) {
            settingList.add(new InterpreterSettingsList(setting.getId(), setting.getName(), setting.getInterpreterInfos(), false));
        }
    }
    return settingList;
}

Example 19 with InterpreterSetting

use of org.apache.zeppelin.interpreter.InterpreterSetting in project zeppelin by apache.

the class InterpreterRestApiTest method testSettingsCRUD.

@Test
public void testSettingsCRUD() throws IOException {
    // when: call create setting API
    String rawRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + "\"dependencies\":[]," + "\"option\": { \"remote\": true, \"session\": false }}";
    JsonObject jsonRequest = gson.fromJson(rawRequest, JsonElement.class).getAsJsonObject();
    PostMethod post = httpPost("/interpreter/setting/", jsonRequest.toString());
    String postResponse = post.getResponseBodyAsString();
    LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
    InterpreterSetting created = convertResponseToInterpreterSetting(postResponse);
    String newSettingId = created.getId();
    // then : call create setting API
    assertThat("test create method:", post, isAllowed());
    post.releaseConnection();
    // when: call read setting API
    GetMethod get = httpGet("/interpreter/setting/" + newSettingId);
    String getResponse = get.getResponseBodyAsString();
    LOG.info("testSettingCRUD get response\n" + getResponse);
    InterpreterSetting previouslyCreated = convertResponseToInterpreterSetting(getResponse);
    // then : read Setting API
    assertThat("Test get method:", get, isAllowed());
    assertEquals(newSettingId, previouslyCreated.getId());
    get.releaseConnection();
    // when: call update setting API
    jsonRequest.getAsJsonObject("properties").addProperty("propname2", "this is new prop");
    PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest.toString());
    LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
    // then: call update setting API
    assertThat("test update method:", put, isAllowed());
    put.releaseConnection();
    // when: call delete setting API
    DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
    LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
    // then: call delete setting API
    assertThat("Test delete method:", delete, isAllowed());
    delete.releaseConnection();
}

Example 20 with InterpreterSetting

use of org.apache.zeppelin.interpreter.InterpreterSetting in project zeppelin by apache.

the class InterpreterRestApiTest method testInterpreterRestart.

@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
    // when: create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph(AuthenticationInfo.ANONYMOUS);
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);
    // when: run markdown paragraph
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
        Thread.sleep(100);
    }
    assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown"));
    // when: restart interpreter
    for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId())) {
        if (setting.getName().equals("md")) {
            // call restart interpreter API
            PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
            assertThat("test interpreter restart:", put, isAllowed());
            put.releaseConnection();
            break;
        }
    }
    // when: run markdown paragraph, again
    p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
    p.setConfig(config);
    p.setText("%md markdown restarted");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
        Thread.sleep(100);
    }
    // then
    assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown restarted"));
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}