use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.
the class DefaultLivyClient method getStatement.
@Override
public Statement getStatement(JerseyRestClient client, SparkLivyProcess sparkLivyProcess, Integer statementId) {
Integer sessionId = sparkLivyProcess.getSessionId();
try {
Statement statement = client.get(STATEMENT_URL(sessionId, statementId), Statement.class);
livyServer.setLivyServerStatus(LivyServerStatus.alive);
logger.debug("statement={}", statement);
return statement;
} catch (WebApplicationException wae) {
if (wae.getResponse().getStatus() != 404 || wae.getResponse().getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) {
livyServer.setLivyServerStatus(LivyServerStatus.http_error);
}
throw wae;
} catch (LivyException le) {
livyServer.setLivyServerStatus(LivyServerStatus.http_error);
throw le;
}
}
use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.
the class LivyRestModelTransformer method toTransformResponse.
public static TransformResponse toTransformResponse(Statement statement, String transformId) {
TransformResponse transformResponse = prepTransformResponse(statement, transformId);
if (transformResponse.getStatus() == TransformResponse.Status.SUCCESS) {
String code = statement.getCode().trim();
if (code.endsWith("dfResultsAsJson")) {
transformResponse.setResults(toTransformQueryResultWithSchema(transformResponse, statement.getOutput()));
} else if (code.endsWith("dfProf")) {
List<OutputRow> rows = toTransformResponseProfileStats(statement.getOutput());
transformResponse.setProfile(toTransformResponseProfileStats(statement.getOutput()));
transformResponse.setActualCols(1);
Integer actualRows = rows.stream().filter(metric -> metric.getMetricType().equals(MetricType.TOTAL_COUNT.toString())).map(metric -> Integer.valueOf(metric.getMetricValue())).findFirst().orElse(1);
transformResponse.setActualRows(actualRows);
transformResponse.setResults(emptyResult());
} else if (code.endsWith("transformAsStr")) {
/* expects that 'statement' contains a payload of TransformResponse in JSON format */
TransformResponse tr = serializeStatementOutputResponse(checkCodeWasWellFormed(statement.getOutput()), TransformResponse.class);
statementIdCache.put(tr.getTable(), statement.getId());
return tr;
} else {
logger.error("Exception Processing Result: ", new LivyCodeException("Unsupported result type requested of Livy. Results not recognized"));
throw new LivyUserException("livy.unsupported_result_type");
}
// end if
}
return transformResponse;
}
use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.
the class SparkLivyRestClient method createJob.
@Override
public SparkJobResponse createJob(@Nonnull final SparkShellProcess process, @Nonnull final SparkJobRequest request) {
logger.entry(process, request);
// Execute job script
final JerseyRestClient client = sparkLivyProcessManager.getClient(process);
final String script = scalaScriptService.wrapScriptForLivy(request);
final Statement statement = submitCode(client, script, process);
final String jobId = ScalaScriptService.newTableName();
sparkLivyProcessManager.setStatementId(jobId, statement.getId());
// Generate response
final SparkJobResponse response = new SparkJobResponse();
response.setId(jobId);
response.setStatus(StatementStateTranslator.translate(statement.getState()));
return logger.exit(response);
}
use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.
the class SparkLivyRestClient method getTransformResult.
@Nonnull
@Override
public Optional<TransformResponse> getTransformResult(@Nonnull SparkShellProcess process, @Nonnull String transformId) {
logger.entry(process, transformId);
Validate.isInstanceOf(SparkLivyProcess.class, process, "SparkLivyRestClient.getTransformResult called on non Livy Process");
SparkLivyProcess sparkLivyProcess = (SparkLivyProcess) process;
JerseyRestClient client = sparkLivyProcessManager.getClient(process);
Integer stmtId = sparkLivyProcessManager.getStatementId(transformId);
Statement statement = livyClient.getStatement(client, sparkLivyProcess, stmtId);
sparkLivyProcessManager.setStatementId(transformId, statement.getId());
TransformResponse response = LivyRestModelTransformer.toTransformResponse(statement, transformId);
if (statement.getState() == StatementState.available && response.getStatus() == TransformResponse.Status.PENDING) {
// TODO:: change this so that if transformId is a livyQueryID it knows what to do. similar to saveResponse. A good first stab at this is at KYLO-2639-withLivyId
// The result came back from Livy, but the transform result still needs to be fetched
String script = scriptGenerator.script("getTransform", ScalaScriptUtils.scalaStr(transformId));
statement = submitCode(client, script, process);
// associate transformId to this new query and get results on subsequent call
sparkLivyProcessManager.setStatementId(transformId, statement.getId());
} else if (response.getStatus() == TransformResponse.Status.ERROR) {
logger.error(String.format("Unexpected error found in transform response:\n%s", response.getMessage()));
throw new LivyUserException("livy.transform_error");
}
// end if
return logger.exit(Optional.of(response));
}
use of com.thinkbiganalytics.kylo.spark.model.Statement in project kylo by Teradata.
the class SparkLivyRestClient method getJobResult.
@Nonnull
@Override
public Optional<SparkJobResponse> getJobResult(@Nonnull final SparkShellProcess process, @Nonnull final String id) {
logger.entry(process, id);
Validate.isInstanceOf(SparkLivyProcess.class, process, "SparkLivyRestClient.getJobResult called on non Livy Process");
SparkLivyProcess sparkLivyProcess = (SparkLivyProcess) process;
// Request result from Livy
final JerseyRestClient client = sparkLivyProcessManager.getClient(process);
final Integer statementId = sparkLivyProcessManager.getStatementId(id);
final Statement statement = livyClient.getStatement(client, sparkLivyProcess, statementId);
sparkLivyProcessManager.setStatementId(id, statement.getId());
// Generate response
final SparkJobResponse response = LivyRestModelTransformer.toJobResponse(id, statement);
if (response.getStatus() != TransformResponse.Status.ERROR) {
return logger.exit(Optional.of(response));
} else {
throw logger.throwing(new SparkException(String.format("Unexpected error found in transform response:\n%s", response.getMessage())));
}
}
Aggregations