Examples with Paragraph org.apache.zeppelin.notebook.Paragraph used on opensource projects

Example 51 with Paragraph

use of org.apache.zeppelin.notebook.Paragraph in project zeppelin by apache.

the class ZeppelinRestApiTest method testNoteJobs.

@Test
public void testNoteJobs() throws IOException, InterruptedException {
    LOG.info("testNoteJobs");
    // Create note to run test.
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    assertNotNull("can't create new note", note);
    note.setName("note for run test");
    Paragraph paragraph = note.addParagraph(AuthenticationInfo.ANONYMOUS);
    Map config = paragraph.getConfig();
    config.put("enabled", true);
    paragraph.setConfig(config);
    paragraph.setText("%md This is test paragraph.");
    note.persist(anonymous);
    String noteId = note.getId();
    note.runAll();
    // wait until job is finished or timeout.
    int timeout = 1;
    while (!paragraph.isTerminated()) {
        Thread.sleep(1000);
        if (timeout++ > 10) {
            LOG.info("testNoteJobs timeout job.");
            break;
        }
    }
    // Call Run note jobs REST API
    PostMethod postNoteJobs = httpPost("/notebook/job/" + noteId, "");
    assertThat("test note jobs run:", postNoteJobs, isAllowed());
    postNoteJobs.releaseConnection();
    // Call Stop note jobs REST API
    DeleteMethod deleteNoteJobs = httpDelete("/notebook/job/" + noteId);
    assertThat("test note stop:", deleteNoteJobs, isAllowed());
    deleteNoteJobs.releaseConnection();
    Thread.sleep(1000);
    // Call Run paragraph REST API
    PostMethod postParagraph = httpPost("/notebook/job/" + noteId + "/" + paragraph.getId(), "");
    assertThat("test paragraph run:", postParagraph, isAllowed());
    postParagraph.releaseConnection();
    Thread.sleep(1000);
    // Call Stop paragraph REST API
    DeleteMethod deleteParagraph = httpDelete("/notebook/job/" + noteId + "/" + paragraph.getId());
    assertThat("test paragraph stop:", deleteParagraph, isAllowed());
    deleteParagraph.releaseConnection();
    Thread.sleep(1000);
    //cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}

Example 52 with Paragraph

use of org.apache.zeppelin.notebook.Paragraph in project zeppelin by apache.

the class ZeppelinRestApiTest method testImportNotebook.

@Test
public void testImportNotebook() throws IOException {
    Map<String, Object> resp;
    String noteName = "source note for import";
    LOG.info("testImortNote");
    // create test note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    assertNotNull("can't create new note", note);
    note.setName(noteName);
    Paragraph paragraph = note.addParagraph(AuthenticationInfo.ANONYMOUS);
    Map config = paragraph.getConfig();
    config.put("enabled", true);
    paragraph.setConfig(config);
    paragraph.setText("%md This is my new paragraph in my new note");
    note.persist(anonymous);
    String sourceNoteId = note.getId();
    // get note content as JSON
    String oldJson = getNoteContent(sourceNoteId);
    // call note post
    PostMethod importPost = httpPost("/notebook/import/", oldJson);
    assertThat(importPost, isAllowed());
    resp = gson.fromJson(importPost.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
    }.getType());
    String importId = (String) resp.get("body");
    assertNotNull("Did not get back a note id in body", importId);
    Note newNote = ZeppelinServer.notebook.getNote(importId);
    assertEquals("Compare note names", noteName, newNote.getName());
    assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs().size());
    // cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
    ZeppelinServer.notebook.removeNote(newNote.getId(), anonymous);
    importPost.releaseConnection();
}

Example 53 with Paragraph

use of org.apache.zeppelin.notebook.Paragraph in project zeppelin by apache.

the class ZeppelinSparkClusterTest method pySparkTest.

@Test
public void pySparkTest() throws IOException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.setName("note");
    int sparkVersion = getSparkVersionNumber(note);
    if (isPyspark() && sparkVersion >= 12) {
        // pyspark supported from 1.2.1
        // run markdown paragraph, again
        Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
        Map config = p.getConfig();
        config.put("enabled", true);
        p.setConfig(config);
        p.setText("%pyspark print(sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b))");
        p.setAuthenticationInfo(anonymous);
        note.run(p.getId());
        waitForFinish(p);
        assertEquals(Status.FINISHED, p.getStatus());
        assertEquals("55\n", p.getResult().message().get(0).getData());
        if (sparkVersion >= 13) {
            // run sqlContext test
            p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
            config = p.getConfig();
            config.put("enabled", true);
            p.setConfig(config);
            p.setText("%pyspark from pyspark.sql import Row\n" + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + "df.collect()");
            p.setAuthenticationInfo(anonymous);
            note.run(p.getId());
            waitForFinish(p);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData());
            // test display Dataframe
            p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
            config = p.getConfig();
            config.put("enabled", true);
            p.setConfig(config);
            p.setText("%pyspark from pyspark.sql import Row\n" + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + "z.show(df)");
            p.setAuthenticationInfo(anonymous);
            note.run(p.getId());
            waitForFinish(p);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType());
            // TODO (zjffdu), one more \n is appended, need to investigate why.
            assertEquals("age\tid\n20\t1\n", p.getResult().message().get(0).getData());
            // test udf
            p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
            config = p.getConfig();
            config.put("enabled", true);
            p.setConfig(config);
            p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + "sqlContext.sql(\"select f1(\\\"abc\\\") as len\").collect()");
            p.setAuthenticationInfo(anonymous);
            note.run(p.getId());
            waitForFinish(p);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("[Row(len=u'3')]\n", p.getResult().message().get(0).getData());
        }
        if (sparkVersion >= 20) {
            // run SparkSession test
            p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
            config = p.getConfig();
            config.put("enabled", true);
            p.setConfig(config);
            p.setText("%pyspark from pyspark.sql import Row\n" + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + "df.collect()");
            p.setAuthenticationInfo(anonymous);
            note.run(p.getId());
            waitForFinish(p);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData());
            // test udf
            p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
            config = p.getConfig();
            config.put("enabled", true);
            p.setConfig(config);
            // use SQLContext to register UDF but use this UDF through SparkSession
            p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + "spark.sql(\"select f1(\\\"abc\\\") as len\").collect()");
            p.setAuthenticationInfo(anonymous);
            note.run(p.getId());
            waitForFinish(p);
            assertEquals(Status.FINISHED, p.getStatus());
            assertEquals("[Row(len=u'3')]\n", p.getResult().message().get(0).getData());
        }
    }
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}

Example 54 with Paragraph

use of org.apache.zeppelin.notebook.Paragraph in project zeppelin by apache.

the class ZeppelinSparkClusterTest method getSparkVersionNumber.

/**
     * Get spark version number as a numerical value.
     * eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ...
     */
private int getSparkVersionNumber(Note note) {
    Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
    note.setName("note");
    Map config = p.getConfig();
    config.put("enabled", true);
    p.setConfig(config);
    p.setText("%spark print(sc.version)");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    waitForFinish(p);
    assertEquals(Status.FINISHED, p.getStatus());
    String sparkVersion = p.getResult().message().get(0).getData();
    System.out.println("Spark version detected " + sparkVersion);
    String[] split = sparkVersion.split("\\.");
    int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]);
    return version;
}

Example 55 with Paragraph

use of org.apache.zeppelin.notebook.Paragraph in project zeppelin by apache.

the class ZeppelinSparkClusterTest method pySparkDepLoaderTest.

@Test
public void pySparkDepLoaderTest() throws IOException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    int sparkVersionNumber = getSparkVersionNumber(note);
    if (isPyspark() && sparkVersionNumber >= 14) {
        // restart spark interpreter
        List<InterpreterSetting> settings = ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
        for (InterpreterSetting setting : settings) {
            if (setting.getName().equals("spark")) {
                ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
                break;
            }
        }
        // load dep
        Paragraph p0 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
        Map config = p0.getConfig();
        config.put("enabled", true);
        p0.setConfig(config);
        p0.setText("%dep z.load(\"com.databricks:spark-csv_2.11:1.2.0\")");
        p0.setAuthenticationInfo(anonymous);
        note.run(p0.getId());
        waitForFinish(p0);
        assertEquals(Status.FINISHED, p0.getStatus());
        // write test csv file
        File tmpFile = File.createTempFile("test", "csv");
        FileUtils.write(tmpFile, "a,b\n1,2");
        // load data using libraries from dep loader
        Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
        p1.setConfig(config);
        String sqlContextName = "sqlContext";
        if (sparkVersionNumber >= 20) {
            sqlContextName = "spark";
        }
        p1.setText("%pyspark\n" + "from pyspark.sql import SQLContext\n" + "print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" + ".load('" + tmpFile.getAbsolutePath() + "').count())");
        p1.setAuthenticationInfo(anonymous);
        note.run(p1.getId());
        waitForFinish(p1);
        assertEquals(Status.FINISHED, p1.getStatus());
        assertEquals("2\n", p1.getResult().message().get(0).getData());
    }
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}