Examples with ResourceSet org.apache.zeppelin.resource.ResourceSet used on opensource projects

Example 11 with ResourceSet

use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.

the class ApplicationLoader method findRequiredResourceSet.

static ResourceSet findRequiredResourceSet(String[][] requiredResources, String noteId, String paragraphId, ResourceSet resources) {
    ResourceSet args = new ResourceSet();
    if (requiredResources == null || requiredResources.length == 0) {
        return args;
    }
    resources = resources.filterByNoteId(noteId).filterByParagraphId(paragraphId);
    for (String[] requires : requiredResources) {
        args.clear();
        for (String require : requires) {
            boolean found = false;
            for (Resource r : resources) {
                if (require.startsWith(":") && r.getClassName().equals(require.substring(1))) {
                    found = true;
                } else if (r.getResourceId().getName().equals(require)) {
                    found = true;
                }
                if (found) {
                    args.add(r);
                    break;
                }
            }
            if (found == false) {
                break;
            }
        }
        if (args.size() == requires.length) {
            return args;
        }
    }
    return null;
}

Example 12 with ResourceSet

use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.

the class ApplicationLoader method load.

/**
 * Instantiate application
 *
 * @param packageInfo
 * @param context
 * @return
 * @throws Exception
 */
public Application load(HeliumPackage packageInfo, ApplicationContext context) throws Exception {
    if (packageInfo.getType() != HeliumType.APPLICATION) {
        throw new ApplicationException("Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader");
    }
    // check if already loaded
    RunningApplication key = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId());
    // get resource required by this package
    ResourceSet resources = findRequiredResourceSet(packageInfo.getResources(), context.getNoteId(), context.getParagraphId());
    // load class
    Class<Application> appClass = loadClass(packageInfo);
    // instantiate
    ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
    ClassLoader cl = appClass.getClassLoader();
    Thread.currentThread().setContextClassLoader(cl);
    try {
        Constructor<Application> constructor = appClass.getConstructor(ApplicationContext.class);
        Application app = new ClassLoaderApplication(constructor.newInstance(context), cl);
        return app;
    } catch (Exception e) {
        throw new ApplicationException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(oldcl);
    }
}

Example 13 with ResourceSet

use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.

the class RemoteInterpreterEventClient method getAllResources.

/**
 * Get all resources except for specific resourcePool
 *
 * @return
 */
@Override
public ResourceSet getAllResources() {
    try {
        List<String> resources = callRemoteFunction(client -> client.getAllResources(intpGroupId));
        ResourceSet resourceSet = new ResourceSet();
        for (String res : resources) {
            RemoteResource resource = RemoteResource.fromJson(res);
            resource.setResourcePoolConnector(this);
            resourceSet.add(resource);
        }
        return resourceSet;
    } catch (Exception e) {
        LOGGER.warn("Fail to getAllResources", e);
        return null;
    }
}

Example 14 with ResourceSet

use of org.apache.zeppelin.resource.ResourceSet in project zeppelin by apache.

the class Helium method suggestApp.

public HeliumPackageSuggestion suggestApp(Paragraph paragraph) {
    HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion();
    Interpreter intp = null;
    try {
        intp = paragraph.getBindedInterpreter();
    } catch (InterpreterNotFoundException e) {
        return suggestion;
    }
    ResourcePool resourcePool = intp.getInterpreterGroup().getResourcePool();
    ResourceSet allResources;
    if (resourcePool != null) {
        if (resourcePool instanceof DistributedResourcePool) {
            allResources = ((DistributedResourcePool) resourcePool).getAll(true);
        } else {
            allResources = resourcePool.getAll();
        }
    } else {
        allResources = interpreterSettingManager.getAllResources();
    }
    for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) {
        for (HeliumPackageSearchResult pkg : pkgs) {
            if (pkg.getPkg().getType() == HeliumType.APPLICATION && pkg.isEnabled()) {
                ResourceSet resources = ApplicationLoader.findRequiredResourceSet(pkg.getPkg().getResources(), paragraph.getNote().getId(), paragraph.getId(), allResources);
                if (resources == null) {
                    continue;
                } else {
                    suggestion.addAvailablePackage(pkg);
                }
                break;
            }
        }
    }
    suggestion.sort();
    return suggestion;
}

Example 15 with ResourceSet

use of org.apache.zeppelin.resource.ResourceSet in project SSM by Intel-bigdata.

the class RemoteInterpreterEventPoller method run.

@Override
public void run() {
    Client client = null;
    AppendOutputRunner runner = new AppendOutputRunner(listener);
    ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
    while (!shutdown) {
        // wait and retry
        if (!interpreterProcess.isRunning()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
            // nothing to do
            }
            continue;
        }
        try {
            client = interpreterProcess.getClient();
        } catch (Exception e1) {
            logger.error("Can't get RemoteInterpreterEvent", e1);
            waitQuietly();
            continue;
        }
        RemoteInterpreterEvent event = null;
        boolean broken = false;
        try {
            event = client.getEvent();
        } catch (TException e) {
            broken = true;
            logger.error("Can't get RemoteInterpreterEvent", e);
            waitQuietly();
            continue;
        } finally {
            interpreterProcess.releaseClient(client, broken);
        }
        Gson gson = new Gson();
        AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
        try {
            if (event.getType() == RemoteInterpreterEventType.NO_OP) {
                continue;
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                angularObjectRegistry.add(angularObject.getName(), angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                AngularObject localAngularObject = angularObjectRegistry.get(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
                if (localAngularObject instanceof RemoteAngularObject) {
                    // to avoid ping-pong loop
                    ((RemoteAngularObject) localAngularObject).set(angularObject.get(), true, false);
                } else {
                    localAngularObject.set(angularObject.get());
                }
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
                InterpreterContextRunner runnerFromRemote = gson.fromJson(event.getData(), RemoteInterpreterContextRunner.class);
                listener.onRemoteRunParagraph(runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
                ResourceSet resourceSet = getAllResourcePoolExcept();
                sendResourcePoolResponseGetAll(resourceSet);
            } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
                String resourceIdString = event.getData();
                ResourceId resourceId = gson.fromJson(resourceIdString, ResourceId.class);
                logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
                Object o = getResource(resourceId);
                sendResourceResponseGet(resourceId, o);
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
                // on output append
                Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputAppend.get("noteId");
                String paragraphId = (String) outputAppend.get("paragraphId");
                int index = Integer.parseInt(outputAppend.get("index"));
                String outputToAppend = (String) outputAppend.get("data");
                String appId = (String) outputAppend.get("appId");
                if (appId == null) {
                    runner.appendBuffer(noteId, paragraphId, index, outputToAppend);
                } else {
                    appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend);
                }
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) {
                Map<String, Object> outputUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputUpdate.get("noteId");
                String paragraphId = (String) outputUpdate.get("paragraphId");
                // clear the output
                listener.onOutputClear(noteId, paragraphId);
                List<Map<String, String>> messages = (List<Map<String, String>>) outputUpdate.get("messages");
                if (messages != null) {
                    for (int i = 0; i < messages.size(); i++) {
                        Map<String, String> m = messages.get(i);
                        InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) m.get("type"));
                        String outputToUpdate = (String) m.get("data");
                        listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate);
                    }
                }
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
                // on output update
                Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputAppend.get("noteId");
                String paragraphId = (String) outputAppend.get("paragraphId");
                int index = Integer.parseInt(outputAppend.get("index"));
                InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) outputAppend.get("type"));
                String outputToUpdate = (String) outputAppend.get("data");
                String appId = (String) outputAppend.get("appId");
                if (appId == null) {
                    listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate);
                } else {
                    appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate);
                }
            } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) {
                // on output update
                Map<String, String> appStatusUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
                }.getType());
                String noteId = appStatusUpdate.get("noteId");
                String paragraphId = appStatusUpdate.get("paragraphId");
                String appId = appStatusUpdate.get("appId");
                String status = appStatusUpdate.get("status");
                appListener.onStatusChange(noteId, paragraphId, appId, status);
            } else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) {
                RemoteZeppelinServerResource reqResourceBody = gson.fromJson(event.getData(), RemoteZeppelinServerResource.class);
                progressRemoteZeppelinControlEvent(reqResourceBody.getResourceType(), listener, reqResourceBody);
            } else if (event.getType() == RemoteInterpreterEventType.META_INFOS) {
                Map<String, String> metaInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
                }.getType());
                String id = interpreterGroup.getId();
                int indexOfColon = id.indexOf(":");
                String settingId = id.substring(0, indexOfColon);
                listener.onMetaInfosReceived(settingId, metaInfos);
            }
            logger.debug("Event from remote process {}", event.getType());
        } catch (Exception e) {
            logger.error("Can't handle event " + event, e);
        }
    }
    if (appendFuture != null) {
        appendFuture.cancel(true);
    }
}