Examples with JerseyRestClient com.thinkbiganalytics.rest.JerseyRestClient used on opensource projects

Example 16 with JerseyRestClient

use of com.thinkbiganalytics.rest.JerseyRestClient in project kylo by Teradata.

the class SparkLivyRestClient method transform.

@Nonnull
@Override
public TransformResponse transform(@Nonnull SparkShellProcess process, @Nonnull TransformRequest request) {
    logger.entry(process, request);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    // a tablename will be calculated for the request
    String transformId = ScalaScriptService.newTableName();
    transformCache.put(request, transformId);
    String script = scalaScriptService.wrapScriptForLivy(request, transformId);
    Statement statement = submitCode(client, script, process);
    // check the server for script result.  If polling limit reach just return to UI with PENDING status.
    statement = pollStatement(client, process, statement.getId(), livyProperties.getPollingLimit());
    sparkLivyProcessManager.setStatementId(transformId, statement.getId());
    return logger.exit(LivyRestModelTransformer.toTransformResponse(statement, transformId));
}

Example 17 with JerseyRestClient

use of com.thinkbiganalytics.rest.JerseyRestClient in project kylo by Teradata.

the class SparkLivyRestClient method getDataSources.

@Nonnull
@Override
public DataSources getDataSources(@Nonnull SparkShellProcess process) {
    logger.entry(process);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    String script = scriptGenerator.script("getDataSources");
    Statement statement = submitCode(client, script, process);
    if (statement.getState() == StatementState.running || statement.getState() == StatementState.waiting) {
        statement = pollStatement(client, process, statement.getId());
    } else {
        throw logger.throwing(new LivyUserException("livy.unexpected_error"));
    }
    return logger.exit(LivyRestModelTransformer.toDataSources(statement));
}

Example 18 with JerseyRestClient

use of com.thinkbiganalytics.rest.JerseyRestClient in project kylo by Teradata.

the class SparkLivyRestClient method saveTransform.

@Nonnull
@Override
public SaveResponse saveTransform(@Nonnull SparkShellProcess process, @Nonnull String transformId, @Nonnull SaveRequest request) {
    logger.entry(process, transformId, request);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    String script = scriptGenerator.wrappedScript("submitSaveJob", "", "\n", ScalaScriptUtils.toJsonInScalaString(request), transformId);
    Statement statement = submitCode(client, script, process);
    statement = pollStatement(client, process, statement.getId());
    SaveResponse saveResponse = LivyRestModelTransformer.toSaveResponse(statement);
    transformIdsToLivyId.put(transformId, statement.getId());
    return logger.exit(saveResponse);
}

Example 19 with JerseyRestClient

use of com.thinkbiganalytics.rest.JerseyRestClient in project kylo by Teradata.

the class SparkLivyProcessManager method startLivySession.

private Session startLivySession(SparkLivyProcess sparkLivyProcess) {
    // it was determined we needed a
    sparkLivyProcess.newSession();
    JerseyRestClient jerseyClient = getClient(sparkLivyProcess);
    Map<String, String> sparkProps = livyProperties.getSparkProperties();
    SessionsPost.Builder builder = new SessionsPost.Builder().kind(// "shared" most likely
    livyProperties.getLivySessionKind().toString()).conf(// "spark.driver.extraJavaOptions", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8990"
    sparkProps);
    logger.debug("LivyProperties={}", livyProperties);
    if (livyProperties.getProxyUser()) {
        String user = processCache.inverse().get(sparkLivyProcess);
        if (user != null) {
            builder.proxyUser(user);
        }
    }
    SessionsPost sessionsPost = builder.build();
    logger.info("sessionsPost={}", sessionsPost);
    Session currentSession;
    try {
        currentSession = livyClient.postSessions(jerseyClient, sessionsPost);
        if (currentSession == null) {
            logger.error("Server not reachable", new LivyServerNotReachableException("Empty result from LivyClient.postSessions"));
            throw new LivyUserException("livy.server_not_found");
        }
    } catch (LivyException le) {
        throw le;
    } catch (Exception e) {
        // resets the latch
        sparkLivyProcess.startFailed();
        this.processStopped(sparkLivyProcess);
        // NOTE: you can get "javax.ws.rs.ProcessingException: java.io.IOException: Error writing to server" on Ubuntu see: https://stackoverflow.com/a/39718929/154461
        throw new LivyException(e);
    }
    sparkLivyProcess.setSessionId(currentSession.getId());
    // notifies listeners
    this.processStarted(sparkLivyProcess);
    // begin monitoring this session if configured to do so..
    heartbeatMonitor.monitorSession(sparkLivyProcess);
    return currentSession;
}

Example 20 with JerseyRestClient

use of com.thinkbiganalytics.rest.JerseyRestClient in project kylo by Teradata.

the class SparkLivyProcessManager method start.

public void start(@Nonnull final SparkLivyProcess sparkLivyProcess) {
    JerseyRestClient jerseyClient = getClient(sparkLivyProcess);
    // fetch or create new server session
    Session currentSession;
    if (sparkLivyProcess.getSessionId() != null) {
        Optional<Session> optSession = getLivySession(sparkLivyProcess);
        if (optSession.isPresent()) {
            currentSession = optSession.get();
        } else {
            currentSession = startLivySession(sparkLivyProcess);
        }
    } else {
        currentSession = startLivySession(sparkLivyProcess);
    }
    Integer currentSessionId = currentSession.getId();
    if (!currentSession.getState().equals(SessionState.idle)) {
        logger.debug("Created session with id='{}', but it was returned with state != idle, state = '{}'", currentSession.getId(), currentSession.getState());
        if (!waitForSessionToBecomeIdle(jerseyClient, currentSessionId)) {
            throw new LivyUserException("livy.start_failed");
        }
        // At this point the server is ready and we can send it an initialization command, any following
        // statement sent by UI will wait for their turn to execute
        initSession(sparkLivyProcess);
    }
    // end if
    // notifies all and any waiting threads session is started, OK to call many times..
    sparkLivyProcess.sessionStarted();
}