Examples with Notebook org.apache.zeppelin.notebook.Notebook used on opensource projects

Example 31 with Notebook

use of org.apache.zeppelin.notebook.Notebook in project zeppelin by apache.

the class NotebookServerTest method bindAngularObjectToRemoteForParagraphs.

@Test
public void bindAngularObjectToRemoteForParagraphs() throws Exception {
    // Given
    final String varName = "name";
    final String value = "DuyHai DOAN";
    final Message messageReceived = new Message(OP.ANGULAR_OBJECT_CLIENT_BIND).put("noteId", "noteId").put("name", varName).put("value", value).put("paragraphId", "paragraphId");
    try {
        final Notebook notebook = mock(Notebook.class);
        notebookServer.setNotebook(() -> notebook);
        notebookServer.setNotebookService(() -> notebookService);
        final Note note = mock(Note.class, RETURNS_DEEP_STUBS);
        when(notebook.processNote(eq("noteId"), Mockito.any())).then(e -> e.getArgumentAt(1, NoteProcessor.class).process(note));
        final Paragraph paragraph = mock(Paragraph.class, RETURNS_DEEP_STUBS);
        when(note.getParagraph("paragraphId")).thenReturn(paragraph);
        final RemoteAngularObjectRegistry mdRegistry = mock(RemoteAngularObjectRegistry.class);
        final InterpreterGroup mdGroup = new InterpreterGroup("mdGroup");
        mdGroup.setAngularObjectRegistry(mdRegistry);
        when(paragraph.getBindedInterpreter().getInterpreterGroup()).thenReturn(mdGroup);
        final AngularObject<String> ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId");
        when(mdRegistry.addAndNotifyRemoteProcess(varName, value, "noteId", "paragraphId")).thenReturn(ao1);
        NotebookSocket conn = mock(NotebookSocket.class);
        NotebookSocket otherConn = mock(NotebookSocket.class);
        final String mdMsg1 = notebookServer.serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao1).put("interpreterGroupId", "mdGroup").put("noteId", "noteId").put("paragraphId", "paragraphId"));
        notebookServer.getConnectionManager().noteSocketMap.put("noteId", asList(conn, otherConn));
        // When
        notebookServer.angularObjectClientBind(conn, messageReceived);
        // Then
        verify(mdRegistry, never()).addAndNotifyRemoteProcess(varName, value, "noteId", null);
        verify(otherConn).send(mdMsg1);
    } finally {
        // reset these so that it won't affect other tests
        notebookServer.setNotebook(() -> NotebookServerTest.notebook);
        notebookServer.setNotebookService(() -> NotebookServerTest.notebookService);
    }
}

Example 32 with Notebook

use of org.apache.zeppelin.notebook.Notebook in project zeppelin by apache.

the class NotebookServerTest method testLoadAngularObjectFromNote.

@Test
public void testLoadAngularObjectFromNote() throws IOException, InterruptedException {
    // create a notebook
    String note1Id = null;
    try {
        note1Id = notebook.createNote("note1", anonymous);
        // get reference to interpreterGroup
        InterpreterGroup interpreterGroup = null;
        List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().get();
        for (InterpreterSetting setting : settings) {
            if (setting.getName().equals("angular")) {
                interpreterGroup = setting.getOrCreateInterpreterGroup("anonymous", note1Id);
                break;
            }
        }
        String p1Id = notebook.processNote(note1Id, note1 -> {
            // start interpreter process
            Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            p1.setText("%angular <h2>Bind here : {{COMMAND_TYPE}}</h2>");
            p1.setAuthenticationInfo(anonymous);
            note1.run(p1.getId());
            return p1.getId();
        });
        // wait for paragraph finished
        Status status = notebook.processNote(note1Id, note1 -> note1.getParagraph(p1Id).getStatus());
        while (true) {
            System.out.println("loop");
            if (status == Job.Status.FINISHED) {
                break;
            }
            Thread.sleep(100);
            status = notebook.processNote(note1Id, note1 -> note1.getParagraph(p1Id).getStatus());
        }
        // sleep for 1 second to make sure job running thread finish to fire event. See ZEPPELIN-3277
        Thread.sleep(1000);
        // set note AngularObject
        AngularObject ao = new AngularObject("COMMAND_TYPE", "COMMAND_TYPE_VALUE", note1Id, p1Id, null);
        notebook.processNote(note1Id, note1 -> {
            note1.addOrUpdateAngularObject("angular-shared_process", ao);
            return null;
        });
        // create sockets and open it
        NotebookSocket sock1 = createWebSocket();
        notebookServer.onOpen(sock1);
        // Check the AngularObjectRegistry of the interpreterGroup before executing GET_NOTE
        Map<String, Map<String, AngularObject>> mapRegistry1 = interpreterGroup.getAngularObjectRegistry().getRegistry();
        assertEquals(0, mapRegistry1.size());
        // open the notebook from sockets, AngularObjectRegistry that triggers the update of the interpreterGroup
        notebookServer.onMessage(sock1, new Message(OP.GET_NOTE).put("id", note1Id).toJson());
        Thread.sleep(1000);
        // After executing GET_NOTE, check the AngularObjectRegistry of the interpreterGroup
        Map<String, Map<String, AngularObject>> mapRegistry2 = interpreterGroup.getAngularObjectRegistry().getRegistry();
        assertEquals(2, mapRegistry1.size());
        AngularObject ao1 = mapRegistry2.get(note1Id + "_" + p1Id).get("COMMAND_TYPE");
        assertEquals("COMMAND_TYPE", ao1.getName());
        assertEquals("COMMAND_TYPE_VALUE", ao1.get());
    } finally {
        if (note1Id != null) {
            notebook.removeNote(note1Id, anonymous);
        }
    }
}

Example 33 with Notebook

use of org.apache.zeppelin.notebook.Notebook in project zeppelin by apache.

the class NotebookRestApiTest method testUpdateParagraphConfig.

@Test
public void testUpdateParagraphConfig() throws IOException {
    LOG.info("Running testUpdateParagraphConfig");
    String noteId = null;
    try {
        noteId = TestUtils.getInstance(Notebook.class).createNote("note1", anonymous);
        String paragraphId = TestUtils.getInstance(Notebook.class).processNote(noteId, note -> {
            Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
            assertNull(p.getConfig().get("colWidth"));
            return p.getId();
        });
        String jsonRequest = "{\"colWidth\": 6.0}";
        CloseableHttpResponse put = httpPut("/notebook/" + noteId + "/paragraph/" + paragraphId + "/config", jsonRequest);
        assertThat("test testUpdateParagraphConfig:", put, isAllowed());
        Map<String, Object> resp = gson.fromJson(EntityUtils.toString(put.getEntity(), StandardCharsets.UTF_8), new TypeToken<Map<String, Object>>() {
        }.getType());
        Map<String, Object> respBody = (Map<String, Object>) resp.get("body");
        Map<String, Object> config = (Map<String, Object>) respBody.get("config");
        put.close();
        assertEquals(config.get("colWidth"), 6.0);
        TestUtils.getInstance(Notebook.class).processNote(noteId, note -> {
            assertEquals(note.getParagraph(paragraphId).getConfig().get("colWidth"), 6.0);
            return null;
        });
    } finally {
        // cleanup
        if (null != noteId) {
            TestUtils.getInstance(Notebook.class).removeNote(noteId, anonymous);
        }
    }
}

Example 34 with Notebook

use of org.apache.zeppelin.notebook.Notebook in project zeppelin by apache.

the class ZSessionIntegrationTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), "helium");
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ALLOWED_ORIGINS.getVarName(), "*");
    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_SESSION_CHECK_INTERVAL.getVarName(), "5000");
    AbstractTestRestApi.startUp(ZSessionIntegrationTest.class.getSimpleName());
    ZeppelinConfiguration zConf = ZeppelinConfiguration.create();
    zConf.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_CLASS.getVarName(), TimeoutLifecycleManager.class.getName());
    zConf.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_TIMEOUT_CHECK_INTERVAL.getVarName(), "5000");
    zConf.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_TIMEOUT_THRESHOLD.getVarName(), "10000");
    notebook = TestUtils.getInstance(Notebook.class);
    sparkHome = DownloadUtils.downloadSpark("2.4.4", "2.7");
    flinkHome = DownloadUtils.downloadFlink("1.12.4", "2.11");
}

Example 35 with Notebook

use of org.apache.zeppelin.notebook.Notebook in project zeppelin by apache.

the class ZeppelinServer method main.

public static void main(String[] args) throws InterruptedException, IOException {
    ZeppelinServer.conf = ZeppelinConfiguration.create();
    jettyWebServer = setupJettyServer(conf);
    initMetrics(conf);
    TimedHandler timedHandler = new TimedHandler(Metrics.globalRegistry, Tags.empty());
    jettyWebServer.setHandler(timedHandler);
    ContextHandlerCollection contexts = new ContextHandlerCollection();
    timedHandler.setHandler(contexts);
    sharedServiceLocator = ServiceLocatorFactory.getInstance().create("shared-locator");
    ServiceLocatorUtilities.enableImmediateScope(sharedServiceLocator);
    ServiceLocatorUtilities.addClasses(sharedServiceLocator, NotebookRepoSync.class, ImmediateErrorHandlerImpl.class);
    ImmediateErrorHandlerImpl handler = sharedServiceLocator.getService(ImmediateErrorHandlerImpl.class);
    ServiceLocatorUtilities.bind(sharedServiceLocator, new AbstractBinder() {

        @Override
        protected void configure() {
            Credentials credentials = new Credentials(conf);
            bindAsContract(InterpreterFactory.class).in(Singleton.class);
            bindAsContract(NotebookRepoSync.class).to(NotebookRepo.class).in(Immediate.class);
            bindAsContract(Helium.class).in(Singleton.class);
            bind(conf).to(ZeppelinConfiguration.class);
            bindAsContract(InterpreterSettingManager.class).in(Singleton.class);
            bindAsContract(InterpreterService.class).in(Singleton.class);
            bind(credentials).to(Credentials.class);
            bindAsContract(GsonProvider.class).in(Singleton.class);
            bindAsContract(WebApplicationExceptionMapper.class).in(Singleton.class);
            bindAsContract(AdminService.class).in(Singleton.class);
            bindAsContract(AuthorizationService.class).in(Singleton.class);
            bindAsContract(ConnectionManager.class).in(Singleton.class);
            bindAsContract(NoteManager.class).in(Singleton.class);
            // TODO(jl): Will make it more beautiful
            if (!StringUtils.isBlank(conf.getShiroPath())) {
                bind(ShiroAuthenticationService.class).to(AuthenticationService.class).in(Singleton.class);
            } else {
                // TODO(jl): Will be added more type
                bind(NoAuthenticationService.class).to(AuthenticationService.class).in(Singleton.class);
            }
            bindAsContract(HeliumBundleFactory.class).in(Singleton.class);
            bindAsContract(HeliumApplicationFactory.class).in(Singleton.class);
            bindAsContract(ConfigurationService.class).in(Singleton.class);
            bindAsContract(NotebookService.class).in(Singleton.class);
            bindAsContract(JobManagerService.class).in(Singleton.class);
            bindAsContract(Notebook.class).in(Singleton.class);
            bindAsContract(NotebookServer.class).to(AngularObjectRegistryListener.class).to(RemoteInterpreterProcessListener.class).to(ApplicationEventListener.class).to(NoteEventListener.class).to(WebSocketServlet.class).in(Singleton.class);
            if (conf.isZeppelinNotebookCronEnable()) {
                bind(QuartzSchedulerService.class).to(SchedulerService.class).in(Singleton.class);
            } else {
                bind(NoSchedulerService.class).to(SchedulerService.class).in(Singleton.class);
            }
            if (conf.getBoolean(ConfVars.ZEPPELIN_SEARCH_ENABLE)) {
                bind(LuceneSearch.class).to(SearchService.class).in(Singleton.class);
            } else {
                bind(NoSearchService.class).to(SearchService.class).in(Singleton.class);
            }
        }
    });
    // Multiple Web UI
    final WebAppContext defaultWebApp = setupWebAppContext(contexts, conf, conf.getString(ConfVars.ZEPPELIN_WAR), conf.getServerContextPath());
    final WebAppContext nextWebApp = setupWebAppContext(contexts, conf, conf.getString(ConfVars.ZEPPELIN_ANGULAR_WAR), WEB_APP_CONTEXT_NEXT);
    initWebApp(defaultWebApp);
    initWebApp(nextWebApp);
    // Cluster Manager Server
    setupClusterManagerServer(sharedServiceLocator);
    // JMX Enable
    if (conf.isJMXEnabled()) {
        int port = conf.getJMXPort();
        // Setup JMX
        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
        jettyWebServer.addBean(mbeanContainer);
        JMXServiceURL jmxURL = new JMXServiceURL(String.format("service:jmx:rmi://0.0.0.0:%d/jndi/rmi://0.0.0.0:%d/jmxrmi", port, port));
        ConnectorServer jmxServer = new ConnectorServer(jmxURL, "org.eclipse.jetty.jmx:name=rmiconnectorserver");
        jettyWebServer.addBean(jmxServer);
        LOG.info("JMX Enabled with port: {}", port);
    }
    LOG.info("Starting zeppelin server");
    try {
        // Instantiates ZeppelinServer
        jettyWebServer.start();
        List<ErrorData> errorData = handler.waitForAtLeastOneConstructionError(5 * 1000);
        if (errorData.size() > 0 && errorData.get(0).getThrowable() != null) {
            throw new Exception(errorData.get(0).getThrowable());
        }
        if (conf.getJettyName() != null) {
            org.eclipse.jetty.http.HttpGenerator.setJettyVersion(conf.getJettyName());
        }
    } catch (Exception e) {
        LOG.error("Error while running jettyServer", e);
        System.exit(-1);
    }
    LOG.info("Done, zeppelin server started");
    runNoteOnStart(conf);
    Runtime.getRuntime().addShutdownHook(shutdown(conf));
    // Try to get Notebook from ServiceLocator, because Notebook instantiation is lazy, it is
    // created when user open zeppelin in browser if we don't get it explicitly here.
    // Lazy loading will cause paragraph recovery and cron job initialization is delayed.
    Notebook notebook = ServiceLocatorUtilities.getService(sharedServiceLocator, Notebook.class.getName());
    ServiceLocatorUtilities.getService(sharedServiceLocator, SearchService.class.getName());
    ServiceLocatorUtilities.getService(sharedServiceLocator, SchedulerService.class.getName());
    // Try to recover here, don't do it in constructor of Notebook, because it would cause deadlock.
    notebook.recoveryIfNecessary();
    // for graceful shutdown, input any key in console window
    if (System.getenv("ZEPPELIN_IDENT_STRING") == null) {
        try {
            System.in.read();
        } catch (IOException e) {
            LOG.error("Exception in ZeppelinServer while main ", e);
        }
        System.exit(0);
    }
    jettyWebServer.join();
    if (!conf.isRecoveryEnabled()) {
        sharedServiceLocator.getService(InterpreterSettingManager.class).close();
    }
}