use of org.apache.zeppelin.resource.ResourceSet in project SSM by Intel-bigdata.
the class RemoteInterpreterEventPoller method getAllResourcePoolExcept.
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(gson.fromJson(res, Resource.class));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
return resourceSet;
}
use of org.apache.zeppelin.resource.ResourceSet in project SSM by Intel-bigdata.
the class ApplicationLoader method load.
/**
* Instantiate application
*
* @param packageInfo
* @param context
* @return
* @throws Exception
*/
public Application load(HeliumPackage packageInfo, ApplicationContext context) throws Exception {
if (packageInfo.getType() != HeliumPackage.Type.APPLICATION) {
throw new ApplicationException("Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader");
}
// check if already loaded
RunningApplication key = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId());
// get resource required by this package
ResourceSet resources = findRequiredResourceSet(packageInfo.getResources(), context.getNoteId(), context.getParagraphId());
// load class
Class<Application> appClass = loadClass(packageInfo);
// instantiate
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
ClassLoader cl = appClass.getClassLoader();
Thread.currentThread().setContextClassLoader(cl);
try {
Constructor<Application> constructor = appClass.getConstructor(ApplicationContext.class);
Application app = new ClassLoaderApplication(constructor.newInstance(context), cl);
return app;
} catch (Exception e) {
throw new ApplicationException(e);
} finally {
Thread.currentThread().setContextClassLoader(oldcl);
}
}
use of org.apache.zeppelin.resource.ResourceSet in project SSM by Intel-bigdata.
the class Helium method suggestApp.
public HeliumPackageSuggestion suggestApp(Paragraph paragraph) {
HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion();
Interpreter intp = paragraph.getCurrentRepl();
if (intp == null) {
return suggestion;
}
ResourcePool resourcePool = intp.getInterpreterGroup().getResourcePool();
ResourceSet allResources;
if (resourcePool != null) {
if (resourcePool instanceof DistributedResourcePool) {
allResources = ((DistributedResourcePool) resourcePool).getAll(true);
} else {
allResources = resourcePool.getAll();
}
} else {
allResources = ResourcePoolUtils.getAllResources();
}
for (List<HeliumPackageSearchResult> pkgs : getAllPackageInfo().values()) {
for (HeliumPackageSearchResult pkg : pkgs) {
if (pkg.getPkg().getType() == HeliumPackage.Type.APPLICATION && pkg.isEnabled()) {
ResourceSet resources = ApplicationLoader.findRequiredResourceSet(pkg.getPkg().getResources(), paragraph.getNote().getId(), paragraph.getId(), allResources);
if (resources == null) {
continue;
} else {
suggestion.addAvailablePackage(pkg);
}
break;
}
}
}
suggestion.sort();
return suggestion;
}
use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.
the class RemoteInterpreterEventPoller method getAllResourcePoolExcept.
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(gson.fromJson(res, Resource.class));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
return resourceSet;
}
use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.
the class RemoteInterpreterEventPoller method run.
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
while (!shutdown) {
// wait and retry
if (!interpreterProcess.isRunning()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// nothing to do
}
continue;
}
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
logger.error("Can't get RemoteInterpreterEvent", e1);
waitQuietly();
continue;
}
RemoteInterpreterEvent event = null;
boolean broken = false;
try {
event = client.getEvent();
} catch (TException e) {
broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
continue;
} finally {
interpreterProcess.releaseClient(client, broken);
}
AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
try {
if (event.getType() == RemoteInterpreterEventType.NO_OP) {
continue;
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
angularObjectRegistry.add(angularObject.getName(), angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
AngularObject localAngularObject = angularObjectRegistry.get(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
if (localAngularObject instanceof RemoteAngularObject) {
// to avoid ping-pong loop
((RemoteAngularObject) localAngularObject).set(angularObject.get(), true, false);
} else {
localAngularObject.set(angularObject.get());
}
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
InterpreterContextRunner runnerFromRemote = gson.fromJson(event.getData(), RemoteInterpreterContextRunner.class);
listener.onRemoteRunParagraph(runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
ResourceSet resourceSet = getAllResourcePoolExcept();
sendResourcePoolResponseGetAll(resourceSet);
} else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
String resourceIdString = event.getData();
ResourceId resourceId = gson.fromJson(resourceIdString, ResourceId.class);
logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
Object o = getResource(resourceId);
sendResourceResponseGet(resourceId, o);
} else if (event.getType() == RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD) {
String message = event.getData();
InvokeResourceMethodEventMessage invokeMethodMessage = gson.fromJson(message, InvokeResourceMethodEventMessage.class);
Object ret = invokeResourceMethod(invokeMethodMessage);
sendInvokeMethodResult(invokeMethodMessage, ret);
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
// on output append
Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
}.getType());
String noteId = (String) outputAppend.get("noteId");
String paragraphId = (String) outputAppend.get("paragraphId");
int index = Integer.parseInt(outputAppend.get("index"));
String outputToAppend = (String) outputAppend.get("data");
String appId = (String) outputAppend.get("appId");
if (appId == null) {
runner.appendBuffer(noteId, paragraphId, index, outputToAppend);
} else {
appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend);
}
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) {
Map<String, Object> outputUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
}.getType());
String noteId = (String) outputUpdate.get("noteId");
String paragraphId = (String) outputUpdate.get("paragraphId");
// clear the output
listener.onOutputClear(noteId, paragraphId);
List<Map<String, String>> messages = (List<Map<String, String>>) outputUpdate.get("messages");
if (messages != null) {
for (int i = 0; i < messages.size(); i++) {
Map<String, String> m = messages.get(i);
InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) m.get("type"));
String outputToUpdate = (String) m.get("data");
listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate);
}
}
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
// on output update
Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
}.getType());
String noteId = (String) outputAppend.get("noteId");
String paragraphId = (String) outputAppend.get("paragraphId");
int index = Integer.parseInt(outputAppend.get("index"));
InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) outputAppend.get("type"));
String outputToUpdate = (String) outputAppend.get("data");
String appId = (String) outputAppend.get("appId");
if (appId == null) {
listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate);
} else {
appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate);
}
} else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) {
// on output update
Map<String, String> appStatusUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
}.getType());
String noteId = appStatusUpdate.get("noteId");
String paragraphId = appStatusUpdate.get("paragraphId");
String appId = appStatusUpdate.get("appId");
String status = appStatusUpdate.get("status");
appListener.onStatusChange(noteId, paragraphId, appId, status);
} else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) {
RemoteZeppelinServerResource reqResourceBody = gson.fromJson(event.getData(), RemoteZeppelinServerResource.class);
progressRemoteZeppelinControlEvent(reqResourceBody.getResourceType(), listener, reqResourceBody);
} else if (event.getType() == RemoteInterpreterEventType.META_INFOS) {
Map<String, String> metaInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
}.getType());
String settingId = RemoteInterpreterUtils.getInterpreterSettingId(interpreterGroup.getId());
listener.onMetaInfosReceived(settingId, metaInfos);
} else if (event.getType() == RemoteInterpreterEventType.PARA_INFOS) {
Map<String, String> paraInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
}.getType());
String noteId = paraInfos.get("noteId");
String paraId = paraInfos.get("paraId");
String settingId = RemoteInterpreterUtils.getInterpreterSettingId(interpreterGroup.getId());
if (noteId != null && paraId != null && settingId != null) {
listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos);
}
}
logger.debug("Event from remote process {}", event.getType());
} catch (Exception e) {
logger.error("Can't handle event " + event, e);
}
}
if (appendFuture != null) {
appendFuture.cancel(true);
}
}
Aggregations