Examples with Statement com.thinkbiganalytics.kylo.spark.model.Statement used on opensource projects

Example 11 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method kyloCatalogTransform.

@Nonnull
public TransformResponse kyloCatalogTransform(@Nonnull final SparkShellProcess process, @Nonnull final KyloCatalogReadRequest request) {
    logger.entry(process, request);
    String script = scriptGenerator.wrappedScript("kyloCatalogTransform", "", "\n", ScalaScriptUtils.toJsonInScalaString(request));
    logger.debug("scala str\n{}", script);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    Statement statement = submitCode(client, script, process);
    if (statement.getState() == StatementState.running || statement.getState() == StatementState.waiting) {
        statement = pollStatement(client, process, statement.getId());
    } else {
        throw logger.throwing(new LivyUserException("livy.unexpected_error"));
    }
    // call with null so a transformId will be generated for this query
    return logger.exit(LivyRestModelTransformer.toTransformResponse(statement, null));
}

Example 12 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method getTransformSave.

@Nonnull
@Override
public Optional<SaveResponse> getTransformSave(@Nonnull SparkShellProcess process, @Nonnull String transformId, @Nonnull String saveId) {
    logger.entry(process, transformId, saveId);
    Validate.isInstanceOf(SparkLivyProcess.class, process, "SparkLivyRestClient.getTransformSave called on non Livy Process");
    SparkLivyProcess sparkLivyProcess = (SparkLivyProcess) process;
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    SaveResponse response;
    Integer stmtId = transformIdsToLivyId.getIfPresent(transformId);
    if (stmtId != null) {
        Statement statement = livyClient.getStatement(client, sparkLivyProcess, stmtId);
        response = LivyRestModelTransformer.toSaveResponse(statement);
        if (statement.getState() == StatementState.available) {
            transformIdsToLivyId.invalidate(transformId);
        }
    } else {
        // saveId is not an integer.  We have already processed Livy Response
        String script = scriptGenerator.script("getSave", saveId);
        Statement statement = submitCode(client, script, process);
        response = LivyRestModelTransformer.toSaveResponse(statement);
        response.setId(saveId);
        // remember the true id.  how? a cache.. it's not gonna get communicated back to us from UI..
        transformIdsToLivyId.put(transformId, statement.getId());
    }
    return logger.exit(Optional.of(response));
}

Example 13 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method transform.

@Nonnull
@Override
public TransformResponse transform(@Nonnull SparkShellProcess process, @Nonnull TransformRequest request) {
    logger.entry(process, request);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    // a tablename will be calculated for the request
    String transformId = ScalaScriptService.newTableName();
    transformCache.put(request, transformId);
    String script = scalaScriptService.wrapScriptForLivy(request, transformId);
    Statement statement = submitCode(client, script, process);
    // check the server for script result.  If polling limit reach just return to UI with PENDING status.
    statement = pollStatement(client, process, statement.getId(), livyProperties.getPollingLimit());
    sparkLivyProcessManager.setStatementId(transformId, statement.getId());
    return logger.exit(LivyRestModelTransformer.toTransformResponse(statement, transformId));
}

Example 14 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method getDataSources.

@Nonnull
@Override
public DataSources getDataSources(@Nonnull SparkShellProcess process) {
    logger.entry(process);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    String script = scriptGenerator.script("getDataSources");
    Statement statement = submitCode(client, script, process);
    if (statement.getState() == StatementState.running || statement.getState() == StatementState.waiting) {
        statement = pollStatement(client, process, statement.getId());
    } else {
        throw logger.throwing(new LivyUserException("livy.unexpected_error"));
    }
    return logger.exit(LivyRestModelTransformer.toDataSources(statement));
}

Example 15 with Statement

use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.

the class SparkLivyRestClient method saveTransform.

@Nonnull
@Override
public SaveResponse saveTransform(@Nonnull SparkShellProcess process, @Nonnull String transformId, @Nonnull SaveRequest request) {
    logger.entry(process, transformId, request);
    JerseyRestClient client = sparkLivyProcessManager.getClient(process);
    String script = scriptGenerator.wrappedScript("submitSaveJob", "", "\n", ScalaScriptUtils.toJsonInScalaString(request), transformId);
    Statement statement = submitCode(client, script, process);
    statement = pollStatement(client, process, statement.getId());
    SaveResponse saveResponse = LivyRestModelTransformer.toSaveResponse(statement);
    transformIdsToLivyId.put(transformId, statement.getId());
    return logger.exit(saveResponse);
}