use of org.apache.zeppelin.user.AuthenticationInfo in project zeppelin by apache.
the class NotebookServer method broadcastReloadedNoteList.
public void broadcastReloadedNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
if (subject == null) {
subject = new AuthenticationInfo(StringUtils.EMPTY);
}
//reload and reply first to requesting user
List<Map<String, String>> notesInfo = generateNotesInfo(true, subject, userAndRoles);
multicastToUser(subject.getUser(), new Message(OP.NOTES_INFO).put("notes", notesInfo));
//to others afterwards
broadcastNoteListExcept(notesInfo, subject);
}
use of org.apache.zeppelin.user.AuthenticationInfo in project zeppelin by apache.
the class LivyInterpreterIT method testSparkInterpreter.
@Test
public void testSparkInterpreter() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setAuthenticationInfo(authInfo).setInterpreterOut(output).build();
sparkInterpreter.open();
LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
interpreterGroup.get("session_1").add(sqlInterpreter);
sqlInterpreter.setInterpreterGroup(interpreterGroup);
sqlInterpreter.open();
try {
// detect spark version
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
boolean isSpark2 = isSpark2(sparkInterpreter, context);
testRDD(sparkInterpreter, isSpark2);
testDataFrame(sparkInterpreter, sqlInterpreter, isSpark2);
} finally {
sparkInterpreter.close();
sqlInterpreter.close();
}
}
use of org.apache.zeppelin.user.AuthenticationInfo in project zeppelin by apache.
the class LivyInterpreterIT method testPySparkInterpreter.
@Test
public void testPySparkInterpreter() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
final LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties);
pysparkInterpreter.setInterpreterGroup(mock(InterpreterGroup.class));
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setAuthenticationInfo(authInfo).setInterpreterOut(output).build();
pysparkInterpreter.open();
// test traceback msg
try {
pysparkInterpreter.getLivyVersion();
// for livy version >=0.3 , input some erroneous spark code, check the shown result is more
// than one line
InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().split("\n").length > 1);
assertTrue(result.message().get(0).getData().contains("Traceback"));
} catch (APINotFoundException e) {
// only livy 0.2 can throw this exception since it doesn't have /version endpoint
// in livy 0.2, most error msg is encapsulated in evalue field, only print(a) in pyspark would
// return none-empty traceback
InterpreterResult result = pysparkInterpreter.interpret("print(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().split("\n").length > 1);
assertTrue(result.message().get(0).getData().contains("Traceback"));
}
// test utf-8 Encoding
String utf8Str = "你你你你你你好";
InterpreterResult reslt = pysparkInterpreter.interpret("print(\"" + utf8Str + "\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, reslt.code());
assertTrue(reslt.message().get(0).getData().contains(utf8Str));
// test special characters
String charStr = "açñiñíûÑoç";
InterpreterResult res = pysparkInterpreter.interpret("print(\"" + charStr + "\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, res.code());
assertTrue(res.message().get(0).getData().contains(charStr));
try {
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
boolean isSpark2 = isSpark2(pysparkInterpreter, context);
// test RDD api
result = pysparkInterpreter.interpret("sc.range(1, 10).sum()", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("45"));
// test DataFrame api
if (!isSpark2) {
pysparkInterpreter.interpret("from pyspark.sql import SQLContext\n" + "sqlContext = SQLContext(sc)", context);
result = pysparkInterpreter.interpret("df=sqlContext.createDataFrame([(\"hello\",20)])\n" + "df.collect()", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
// python2 has u and python3 don't have u
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]") || result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
} else {
result = pysparkInterpreter.interpret("df=spark.createDataFrame([(\"hello\",20)])\n" + "df.collect()", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
// python2 has u and python3 don't have u
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]") || result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
}
// test magic api
pysparkInterpreter.interpret("t = [{\"name\":\"userA\", \"role\":\"roleA\"}," + "{\"name\":\"userB\", \"role\":\"roleB\"}]", context);
result = pysparkInterpreter.interpret("%table t", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("userA"));
// error
result = pysparkInterpreter.interpret("print(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("name 'a' is not defined"));
// cancel
if (pysparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
pysparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = pysparkInterpreter.interpret("import time\n" + "sc.range(1, 10).foreach(lambda a: time.sleep(10))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") || message.contains("Job is cancelled"));
}
} finally {
pysparkInterpreter.close();
}
}
use of org.apache.zeppelin.user.AuthenticationInfo in project zeppelin by apache.
the class LivyInterpreterIT method testRDD.
private void testRDD(final LivySparkInterpreter sparkInterpreter, boolean isSpark2) {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setAuthenticationInfo(authInfo).setInterpreterOut(output).build();
;
InterpreterResult result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("Double = 55.0"));
// single line comment
String singleLineComment = "println(1)// my comment";
result = sparkInterpreter.interpret(singleLineComment, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
// multiple line comment
String multipleLineComment = "println(1)/* multiple \n" + "line \n" + "comment */";
result = sparkInterpreter.interpret(multipleLineComment, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
// multi-line string
String multiLineString = "val str = \"\"\"multiple\n" + "line\"\"\"\n" + "println(str)";
result = sparkInterpreter.interpret(multiLineString, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("multiple\nline"));
// case class
String caseClassCode = "case class Person(id:Int, \n" + "name:String)\n" + "val p=Person(1, \"name_a\")";
result = sparkInterpreter.interpret(caseClassCode, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("p: Person = Person(1,name_a)"));
// object class
String objectClassCode = "object Person {}";
result = sparkInterpreter.interpret(objectClassCode, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
if (!isSpark2) {
assertTrue(result.message().get(0).getData().contains("defined module Person"));
} else {
assertTrue(result.message().get(0).getData().contains("defined object Person"));
}
// html output
String htmlCode = "println(\"%html <h1> hello </h1>\")";
result = sparkInterpreter.interpret(htmlCode, context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
// error
result = sparkInterpreter.interpret("println(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("error: not found: value a"));
// incomplete code
result = sparkInterpreter.interpret("if(true){", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("incomplete statement"));
// cancel
if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkInterpreter.interpret("sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") || message.contains("Job is cancelled"));
}
}
use of org.apache.zeppelin.user.AuthenticationInfo in project zeppelin by apache.
the class LivyInterpreterIT method testSparkRInterpreter.
@Test
public void testSparkRInterpreter() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
final LivySparkRInterpreter sparkRInterpreter = new LivySparkRInterpreter(properties);
sparkRInterpreter.setInterpreterGroup(mock(InterpreterGroup.class));
try {
sparkRInterpreter.getLivyVersion();
} catch (APINotFoundException e) {
// don't run sparkR test for livy 0.2 as there's some issues for livy 0.2
return;
}
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder().setNoteId("noteId").setParagraphId("paragraphId").setAuthenticationInfo(authInfo).setInterpreterOut(output).build();
sparkRInterpreter.open();
try {
// only test it in livy newer than 0.2.0
boolean isSpark2 = isSpark2(sparkRInterpreter, context);
InterpreterResult result = null;
// test DataFrame api
if (isSpark2) {
result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\nhead(df)", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
// cancel
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkRInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\n" + "df1 <- dapplyCollect(df, function(x) " + "{ Sys.sleep(10); x <- cbind(x, x$waiting * 60) })", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") || message.contains("Job is cancelled"));
} else {
result = sparkRInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)" + "\nhead(df)", context);
assertEquals(result.toString(), InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
}
// error
result = sparkRInterpreter.interpret("cat(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("object 'a' not found"));
} finally {
sparkRInterpreter.close();
}
}
Aggregations