Examples with Client org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client used on opensource projects

Example 11 with Client

use of org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client in project zeppelin by apache.

the class RemoteInterpreterTest method should_push_local_angular_repo_to_remote.

@Test
public void should_push_local_angular_repo_to_remote() throws Exception {
    //Given
    final Client client = Mockito.mock(Client.class);
    final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId", MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null, null, "anonymous", false);
    final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
    registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
    final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
    interpreterGroup.setAngularObjectRegistry(registry);
    intr.setInterpreterGroup(interpreterGroup);
    final java.lang.reflect.Type registryType = new TypeToken<Map<String, Map<String, AngularObject>>>() {
    }.getType();
    final Gson gson = new Gson();
    final String expected = gson.toJson(registry.getRegistry(), registryType);
    //When
    intr.pushAngularObjectRegistryToRemote(client);
    //Then
    Mockito.verify(client).angularRegistryPush(expected);
}

Example 12 with Client

use of org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client in project zeppelin by apache.

the class RemoteInterpreterEventPoller method getAllResourcePoolExcept.

private ResourceSet getAllResourcePoolExcept() {
    ResourceSet resourceSet = new ResourceSet();
    for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
        if (intpGroup.getId().equals(interpreterGroup.getId())) {
            continue;
        }
        RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
        if (remoteInterpreterProcess == null) {
            ResourcePool localPool = intpGroup.getResourcePool();
            if (localPool != null) {
                resourceSet.addAll(localPool.getAll());
            }
        } else if (interpreterProcess.isRunning()) {
            Client client = null;
            boolean broken = false;
            try {
                client = remoteInterpreterProcess.getClient();
                List<String> resourceList = client.resourcePoolGetAll();
                Gson gson = new Gson();
                for (String res : resourceList) {
                    resourceSet.add(gson.fromJson(res, Resource.class));
                }
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                broken = true;
            } finally {
                if (client != null) {
                    intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
                }
            }
        }
    }
    return resourceSet;
}

Example 13 with Client

use of org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client in project zeppelin by apache.

the class RemoteInterpreterEventPoller method getResource.

private Object getResource(ResourceId resourceId) {
    InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(resourceId.getResourcePoolId());
    if (intpGroup == null) {
        return null;
    }
    RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
    if (remoteInterpreterProcess == null) {
        ResourcePool localPool = intpGroup.getResourcePool();
        if (localPool != null) {
            return localPool.get(resourceId.getName());
        }
    } else if (interpreterProcess.isRunning()) {
        Client client = null;
        boolean broken = false;
        try {
            client = remoteInterpreterProcess.getClient();
            ByteBuffer res = client.resourceGet(resourceId.getNoteId(), resourceId.getParagraphId(), resourceId.getName());
            Object o = Resource.deserializeObject(res);
            return o;
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            broken = true;
        } finally {
            if (client != null) {
                intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
            }
        }
    }
    return null;
}

Example 14 with Client

use of org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client in project zeppelin by apache.

the class RemoteInterpreterEventPoller method progressRemoteZeppelinControlEvent.

private void progressRemoteZeppelinControlEvent(RemoteZeppelinServerResource.Type resourceType, RemoteInterpreterProcessListener remoteWorksEventListener, RemoteZeppelinServerResource reqResourceBody) throws Exception {
    boolean broken = false;
    final Gson gson = new Gson();
    final String eventOwnerKey = reqResourceBody.getOwnerKey();
    Client interpreterServerMain = null;
    try {
        interpreterServerMain = interpreterProcess.getClient();
        final Client eventClient = interpreterServerMain;
        if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
            final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
            ZeppelinServerResourceParagraphRunner reqRunnerContext = new ZeppelinServerResourceParagraphRunner();
            Map<String, Object> reqResourceMap = (Map<String, Object>) reqResourceBody.getData();
            String noteId = (String) reqResourceMap.get("noteId");
            String paragraphId = (String) reqResourceMap.get("paragraphId");
            reqRunnerContext.setNoteId(noteId);
            reqRunnerContext.setParagraphId(paragraphId);
            RemoteInterpreterProcessListener.RemoteWorksEventListener callBackEvent = new RemoteInterpreterProcessListener.RemoteWorksEventListener() {

                @Override
                public void onFinished(Object resultObject) {
                    boolean clientBroken = false;
                    if (resultObject != null && resultObject instanceof List) {
                        List<InterpreterContextRunner> runnerList = (List<InterpreterContextRunner>) resultObject;
                        for (InterpreterContextRunner r : runnerList) {
                            remoteRunners.add(new ZeppelinServerResourceParagraphRunner(r.getNoteId(), r.getParagraphId()));
                        }
                        final RemoteZeppelinServerResource resResource = new RemoteZeppelinServerResource();
                        resResource.setOwnerKey(eventOwnerKey);
                        resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
                        resResource.setData(remoteRunners);
                        try {
                            eventClient.onReceivedZeppelinResource(gson.toJson(resResource));
                        } catch (Exception e) {
                            clientBroken = true;
                            logger.error("Can't get RemoteInterpreterEvent", e);
                            waitQuietly();
                        } finally {
                            interpreterProcess.releaseClient(eventClient, clientBroken);
                        }
                    }
                }

                @Override
                public void onError() {
                    logger.info("onGetParagraphRunners onError");
                }
            };
            remoteWorksEventListener.onGetParagraphRunners(reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
        }
    } catch (Exception e) {
        broken = true;
        logger.error("Can't get RemoteInterpreterEvent", e);
        waitQuietly();
    } finally {
        interpreterProcess.releaseClient(interpreterServerMain, broken);
    }
}

Example 15 with Client

use of org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client in project zeppelin by apache.

the class RemoteInterpreterEventPoller method run.

@Override
public void run() {
    Client client = null;
    AppendOutputRunner runner = new AppendOutputRunner(listener);
    ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
    while (!shutdown) {
        // wait and retry
        if (!interpreterProcess.isRunning()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
            // nothing to do
            }
            continue;
        }
        try {
            client = interpreterProcess.getClient();
        } catch (Exception e1) {
            logger.error("Can't get RemoteInterpreterEvent", e1);
            waitQuietly();
            continue;
        }
        RemoteInterpreterEvent event = null;
        boolean broken = false;
        try {
            event = client.getEvent();
        } catch (TException e) {
            broken = true;
            logger.error("Can't get RemoteInterpreterEvent", e);
            waitQuietly();
            continue;
        } finally {
            interpreterProcess.releaseClient(client, broken);
        }
        AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
        try {
            if (event.getType() == RemoteInterpreterEventType.NO_OP) {
                continue;
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                angularObjectRegistry.add(angularObject.getName(), angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                AngularObject localAngularObject = angularObjectRegistry.get(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
                if (localAngularObject instanceof RemoteAngularObject) {
                    // to avoid ping-pong loop
                    ((RemoteAngularObject) localAngularObject).set(angularObject.get(), true, false);
                } else {
                    localAngularObject.set(angularObject.get());
                }
            } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
                AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
                angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
                InterpreterContextRunner runnerFromRemote = gson.fromJson(event.getData(), RemoteInterpreterContextRunner.class);
                listener.onRemoteRunParagraph(runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
            } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
                ResourceSet resourceSet = getAllResourcePoolExcept();
                sendResourcePoolResponseGetAll(resourceSet);
            } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
                String resourceIdString = event.getData();
                ResourceId resourceId = gson.fromJson(resourceIdString, ResourceId.class);
                logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
                Object o = getResource(resourceId);
                sendResourceResponseGet(resourceId, o);
            } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD) {
                String message = event.getData();
                InvokeResourceMethodEventMessage invokeMethodMessage = gson.fromJson(message, InvokeResourceMethodEventMessage.class);
                Object ret = invokeResourceMethod(invokeMethodMessage);
                sendInvokeMethodResult(invokeMethodMessage, ret);
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
                // on output append
                Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputAppend.get("noteId");
                String paragraphId = (String) outputAppend.get("paragraphId");
                int index = Integer.parseInt(outputAppend.get("index"));
                String outputToAppend = (String) outputAppend.get("data");
                String appId = (String) outputAppend.get("appId");
                if (appId == null) {
                    runner.appendBuffer(noteId, paragraphId, index, outputToAppend);
                } else {
                    appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend);
                }
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) {
                Map<String, Object> outputUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputUpdate.get("noteId");
                String paragraphId = (String) outputUpdate.get("paragraphId");
                // clear the output
                listener.onOutputClear(noteId, paragraphId);
                List<Map<String, String>> messages = (List<Map<String, String>>) outputUpdate.get("messages");
                if (messages != null) {
                    for (int i = 0; i < messages.size(); i++) {
                        Map<String, String> m = messages.get(i);
                        InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) m.get("type"));
                        String outputToUpdate = (String) m.get("data");
                        listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate);
                    }
                }
            } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
                // on output update
                Map<String, String> outputAppend = gson.fromJson(event.getData(), new TypeToken<Map<String, Object>>() {
                }.getType());
                String noteId = (String) outputAppend.get("noteId");
                String paragraphId = (String) outputAppend.get("paragraphId");
                int index = Integer.parseInt(outputAppend.get("index"));
                InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) outputAppend.get("type"));
                String outputToUpdate = (String) outputAppend.get("data");
                String appId = (String) outputAppend.get("appId");
                if (appId == null) {
                    listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate);
                } else {
                    appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate);
                }
            } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) {
                // on output update
                Map<String, String> appStatusUpdate = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
                }.getType());
                String noteId = appStatusUpdate.get("noteId");
                String paragraphId = appStatusUpdate.get("paragraphId");
                String appId = appStatusUpdate.get("appId");
                String status = appStatusUpdate.get("status");
                appListener.onStatusChange(noteId, paragraphId, appId, status);
            } else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) {
                RemoteZeppelinServerResource reqResourceBody = gson.fromJson(event.getData(), RemoteZeppelinServerResource.class);
                progressRemoteZeppelinControlEvent(reqResourceBody.getResourceType(), listener, reqResourceBody);
            } else if (event.getType() == RemoteInterpreterEventType.META_INFOS) {
                Map<String, String> metaInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
                }.getType());
                String settingId = RemoteInterpreterUtils.getInterpreterSettingId(interpreterGroup.getId());
                listener.onMetaInfosReceived(settingId, metaInfos);
            } else if (event.getType() == RemoteInterpreterEventType.PARA_INFOS) {
                Map<String, String> paraInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() {
                }.getType());
                String noteId = paraInfos.get("noteId");
                String paraId = paraInfos.get("paraId");
                String settingId = RemoteInterpreterUtils.getInterpreterSettingId(interpreterGroup.getId());
                if (noteId != null && paraId != null && settingId != null) {
                    listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos);
                }
            }
            logger.debug("Event from remote process {}", event.getType());
        } catch (Exception e) {
            logger.error("Can't handle event " + event, e);
        }
    }
    if (appendFuture != null) {
        appendFuture.cancel(true);
    }
}