Examples with Statement com.thinkbiganalytics.kylo.spark.model.Statement used on opensource projects

Example 6 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method downloadTransform.

@Nonnull
@Override
public Optional<Response> downloadTransform(@Nonnull SparkShellProcess process, @Nonnull String transformId, @Nonnull String saveId) {
    // 1. get "SaveResult" serialized from Livy
    logger.entry(process, transformId, saveId);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    String script = scriptGenerator.script("getSaveResult", ScalaScriptUtils.scalaStr(saveId));
    Statement statement = submitCode(client, script, process);
    if (statement.getState() == StatementState.running || statement.getState() == StatementState.waiting) {
        statement = pollStatement(client, process, statement.getId());
    } else {
        throw logger.throwing(new LivyUserException("livy.unexpected_error"));
    }
    URI uri = LivyRestModelTransformer.toUri(statement);
    SaveResult result = new SaveResult(new Path(uri));
    // 2. Create a response with data from filesysem
    if (result.getPath() != null) {
        Optional<Response> response = Optional.of(Response.ok(new ZipStreamingOutput(result.getPath(), fileSystem)).header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM).header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + saveId + ".zip\"").build());
        return logger.exit(response);
    } else {
        return logger.exit(Optional.of(createErrorResponse(Response.Status.NOT_FOUND, "download.notFound")));
    }
}

Example 7 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyProcessManager method initSession.

private void initSession(SparkLivyProcess sparkLivyProcess) {
    JerseyRestClient jerseyClient = getClient(sparkLivyProcess);
    String script = scalaScriptService.getInitSessionScript();
    StatementsPost sp = new StatementsPost.Builder().kind(StatementKind.spark.toString()).code(script).build();
    Statement statement = livyClient.postStatement(jerseyClient, sparkLivyProcess, sp);
    // NOTE:  why pollStatement now?  so we block on result.
    livyClient.pollStatement(jerseyClient, sparkLivyProcess, statement.getId());
}

Example 8 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class DefaultLivyClient method pollStatement.

@Override
public Statement pollStatement(JerseyRestClient jerseyClient, SparkLivyProcess sparkLivyProcess, Integer stmtId, Long wait) {
    logger.entry(jerseyClient, sparkLivyProcess, stmtId, wait);
    long stopPolling = Long.MAX_VALUE;
    long startMillis = System.currentTimeMillis();
    if (wait != null) {
        // Limit the amount of time we will poll for a statement to complete.
        stopPolling = startMillis + livyProperties.getPollingLimit();
    }
    Statement statement;
    int pollCount = 1;
    do {
        statement = getStatement(jerseyClient, sparkLivyProcess, stmtId);
        if (statement.getState().equals(StatementState.error)) {
            // TODO: what about cancelled? or cancelling?
            logger.error("Unexpected error encountered while processing a statement", new LivyCodeException(statement.toString()));
            throw logger.throwing(new LivyUserException("livy.unexpected_error"));
        }
        if (System.currentTimeMillis() > stopPolling || statement.getState().equals(StatementState.available)) {
            break;
        }
        logger.trace("Statement was not ready, polling now with attempt '{}'", pollCount++);
        // statement not ready, wait for some time...
        try {
            Thread.sleep(livyProperties.getPollingInterval());
        } catch (InterruptedException e) {
            logger.error("Thread interrupted while polling Livy", e);
        }
    } while (true);
    logger.debug("exit DefaultLivyClient poll statement in '{}' millis, after '{}' attempts ", System.currentTimeMillis() - startMillis, pollCount);
    return logger.exit(statement);
}

Example 9 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class DefaultLivyClient method postStatement.

@Override
public Statement postStatement(JerseyRestClient client, SparkLivyProcess sparkLivyProcess, StatementsPost sp) {
    Integer sessionId = sparkLivyProcess.getSessionId();
    try {
        Statement statement = client.post(STATEMENTS_URL(sessionId), sp, Statement.class);
        livyServer.setLivyServerStatus(LivyServerStatus.alive);
        logger.debug("statement={}", statement);
        return statement;
    } catch (WebApplicationException wae) {
        if (wae.getResponse().getStatus() != 404 || wae.getResponse().getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
            livyServer.setLivyServerStatus(LivyServerStatus.http_error);
        }
        throw wae;
    } catch (LivyException le) {
        livyServer.setLivyServerStatus(LivyServerStatus.http_error);
        throw le;
    }
}

Example 10 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class TestSparkLivyRestClient method testGetLivySessions.

/**
 * This test assumes: 1) you have configured a Livy Server for Kerberos and that it is running at 8998 2) that server has the library 'kylo-spark-shell-client-v1-0.10.0-SNAPSHOT.jar' installed
 */
@Test
// ignore, for now, because this is an integration test that requires livy configured on the test system
@Ignore
public void testGetLivySessions() throws JsonProcessingException, InterruptedException {
    System.setProperty("sun.security.krb5.debug", "true");
    System.setProperty("sun.security.jgss.debug", "true");
    SessionsGet sg = new SessionsGet.Builder().from(1).size(2).build();
    String sgJson = new ObjectMapper().writeValueAsString(sg);
    logger.debug("{}", sgJson);
    assertThat(sgJson).isEqualToIgnoringCase("{\"from\":1,\"size\":2}");
    logger.debug("kerberosSparkProperties={}", kerberosSparkProperties);
    SparkShellProcess sparkProcess = sparkLivyProcessManager.getProcessForUser("kylo");
    // ((SparkLivyProcess) sparkProcess).setPort(8999);
    JerseyRestClient client = sparkLivyProcessManager.getClient(sparkProcess);
    sparkLivyProcessManager.start("kylo");
    Integer stmtId = 0;
    Statement statement = livyRestProvider.pollStatement(client, sparkProcess, stmtId);
    logger.debug("statement={}", statement);
    assertThat(statement.getState()).isEqualTo(StatementState.available);
    assertThat(statement.getOutput().getStatus()).isEqualTo(StatementOutputStatus.ok);
}