Examples with Session com.thinkbiganalytics.kylo.spark.model.Session used on opensource projects

Example 1 with Session

use of com.thinkbiganalytics.kylo.spark.model.Session in project kylo by Teradata.

the class DefaultLivyClient method postSessions.

@Override
public Session postSessions(JerseyRestClient client, SessionsPost sessionsPost) {
    try {
        Session session = client.post(SESSIONS_URL, sessionsPost, Session.class);
        livyServer.setLivyServerStatus(LivyServerStatus.alive);
        livyServer.setSessionIdHighWaterMark(session.getId());
        livyServer.setLivySessionState(session.getId(), session.getState());
        logger.debug("session={}", session);
        return session;
    } catch (WebApplicationException wae) {
        if (wae.getResponse().getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
            livyServer.setLivyServerStatus(LivyServerStatus.http_error);
        }
        throw wae;
    } catch (LivyException le) {
        livyServer.setLivyServerStatus(LivyServerStatus.http_error);
        throw le;
    }
}

Example 2 with Session

use of com.thinkbiganalytics.kylo.spark.model.Session in project kylo by Teradata.

the class SparkLivyProcessManager method waitForSessionToBecomeIdle.

/**
 * returns true is session becomes idle; false if it fails to start
 */
private boolean waitForSessionToBecomeIdle(JerseyRestClient jerseyClient, Integer id) {
    Optional<Session> optSession;
    do {
        try {
            Thread.sleep(livyProperties.getPollingInterval());
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        SessionsGetResponse sessions = livyClient.getSessions(jerseyClient);
        logger.debug("poll server for session with id='{}'", id);
        optSession = sessions.getSessionWithId(id);
        if (optSession.isPresent() && SessionState.FINAL_STATES.contains(optSession.get().getState())) {
            return false;
        }
    } while (!(optSession.isPresent() && optSession.get().getState().equals(SessionState.idle)));
    return true;
}

Example 3 with Session

use of com.thinkbiganalytics.kylo.spark.model.Session in project kylo by Teradata.

the class LivyHeartbeatMonitor method checkSession.

public Optional<SessionState> checkSession(SparkLivyProcess sparkLivyProcess) {
    // TODO: checkSession seems to get in a state where it is called multiple times per heartbeat
    logger.entry(sparkLivyProcess);
    SessionState sessionState = null;
    try {
        Session session = livyClient.getSession(restClient, sparkLivyProcess);
        sessionState = session.getState();
    } catch (ProcessingException pe) {
        logger.trace("Caught ProcessingException:", pe);
        if (!(pe.getCause() instanceof SocketTimeoutException || pe.getCause() instanceof SSLHandshakeException || pe.getCause() instanceof SocketException)) {
            throw logger.throwing(pe);
        }
    } catch (WebApplicationException wae) {
    // recording of state for wae's was taken care of by livyClient.getSession
    }
    return logger.exit(Optional.ofNullable(sessionState));
}

Example 4 with Session

use of com.thinkbiganalytics.kylo.spark.model.Session in project kylo by Teradata.

the class DefaultLivyClient method getSession.

@Override
public Session getSession(JerseyRestClient client, SparkLivyProcess sparkLivyProcess) {
    Integer sessionId = sparkLivyProcess.getSessionId();
    try {
        Session session = client.get(SESSION_URL(sessionId), Session.class);
        livyServer.setLivyServerStatus(LivyServerStatus.alive);
        livyServer.setLivySessionState(session.getId(), session.getState());
        logger.debug("session={}", session);
        return session;
    } catch (WebApplicationException wae) {
        // should catch 404, 500 etc.
        livyServer.recordSessionState(sessionId, wae.getResponse().getStatus(), null);
        throw wae;
    } catch (Exception e) {
        logger.error("Unexpected exception occurred:", e);
        throw e;
    }
}

Example 5 with Session

use of com.thinkbiganalytics.kylo.spark.model.Session in project kylo by Teradata.

the class SparkLivyProcessManager method startLivySession.

private Session startLivySession(SparkLivyProcess sparkLivyProcess) {
    // it was determined we needed a
    sparkLivyProcess.newSession();
    JerseyRestClient jerseyClient = getClient(sparkLivyProcess);
    Map<String, String> sparkProps = livyProperties.getSparkProperties();
    SessionsPost.Builder builder = new SessionsPost.Builder().kind(// "shared" most likely
    livyProperties.getLivySessionKind().toString()).conf(// "spark.driver.extraJavaOptions", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8990"
    sparkProps);
    logger.debug("LivyProperties={}", livyProperties);
    if (livyProperties.getProxyUser()) {
        String user = processCache.inverse().get(sparkLivyProcess);
        if (user != null) {
            builder.proxyUser(user);
        }
    }
    SessionsPost sessionsPost = builder.build();
    logger.info("sessionsPost={}", sessionsPost);
    Session currentSession;
    try {
        currentSession = livyClient.postSessions(jerseyClient, sessionsPost);
        if (currentSession == null) {
            logger.error("Server not reachable", new LivyServerNotReachableException("Empty result from LivyClient.postSessions"));
            throw new LivyUserException("livy.server_not_found");
        }
    } catch (LivyException le) {
        throw le;
    } catch (Exception e) {
        // resets the latch
        sparkLivyProcess.startFailed();
        this.processStopped(sparkLivyProcess);
        // NOTE: you can get "javax.ws.rs.ProcessingException: java.io.IOException: Error writing to server" on Ubuntu see: https://stackoverflow.com/a/39718929/154461
        throw new LivyException(e);
    }
    sparkLivyProcess.setSessionId(currentSession.getId());
    // notifies listeners
    this.processStarted(sparkLivyProcess);
    // begin monitoring this session if configured to do so..
    heartbeatMonitor.monitorSession(sparkLivyProcess);
    return currentSession;
}