use of com.thinkbiganalytics.kylo.spark.exceptions.LivyCodeException in project kylo by Teradata.
the class LivyRestModelTransformer method toTransformResponse.
public static TransformResponse toTransformResponse(Statement statement, String transformId) {
TransformResponse transformResponse = prepTransformResponse(statement, transformId);
if (transformResponse.getStatus() == TransformResponse.Status.SUCCESS) {
String code = statement.getCode().trim();
if (code.endsWith("dfResultsAsJson")) {
transformResponse.setResults(toTransformQueryResultWithSchema(transformResponse, statement.getOutput()));
} else if (code.endsWith("dfProf")) {
List<OutputRow> rows = toTransformResponseProfileStats(statement.getOutput());
transformResponse.setProfile(toTransformResponseProfileStats(statement.getOutput()));
transformResponse.setActualCols(1);
Integer actualRows = rows.stream().filter(metric -> metric.getMetricType().equals(MetricType.TOTAL_COUNT.toString())).map(metric -> Integer.valueOf(metric.getMetricValue())).findFirst().orElse(1);
transformResponse.setActualRows(actualRows);
transformResponse.setResults(emptyResult());
} else if (code.endsWith("transformAsStr")) {
/* expects that 'statement' contains a payload of TransformResponse in JSON format */
TransformResponse tr = serializeStatementOutputResponse(checkCodeWasWellFormed(statement.getOutput()), TransformResponse.class);
statementIdCache.put(tr.getTable(), statement.getId());
return tr;
} else {
logger.error("Exception Processing Result: ", new LivyCodeException("Unsupported result type requested of Livy. Results not recognized"));
throw new LivyUserException("livy.unsupported_result_type");
}
// end if
}
return transformResponse;
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyCodeException in project kylo by Teradata.
the class LivyRestModelTransformer method checkCodeWasWellFormed.
private static StatementOutputResponse checkCodeWasWellFormed(StatementOutputResponse statementOutputResponse) {
if (statementOutputResponse != null && statementOutputResponse.getStatus() != StatementOutputStatus.ok) {
String msg = String.format("Malformed code sent to Livy. ErrorType='%s', Error='%s', Traceback='%s'", statementOutputResponse.getEname(), statementOutputResponse.getEvalue(), statementOutputResponse.getTraceback());
logger.error("Exception Processing Query: ", new LivyCodeException(msg));
throw new LivyUserException("livy.syntax");
}
return statementOutputResponse;
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyCodeException in project kylo by Teradata.
the class DefaultLivyClient method pollStatement.
@Override
public Statement pollStatement(JerseyRestClient jerseyClient, SparkLivyProcess sparkLivyProcess, Integer stmtId, Long wait) {
logger.entry(jerseyClient, sparkLivyProcess, stmtId, wait);
long stopPolling = Long.MAX_VALUE;
long startMillis = System.currentTimeMillis();
if (wait != null) {
// Limit the amount of time we will poll for a statement to complete.
stopPolling = startMillis + livyProperties.getPollingLimit();
}
Statement statement;
int pollCount = 1;
do {
statement = getStatement(jerseyClient, sparkLivyProcess, stmtId);
if (statement.getState().equals(StatementState.error)) {
// TODO: what about cancelled? or cancelling?
logger.error("Unexpected error encountered while processing a statement", new LivyCodeException(statement.toString()));
throw logger.throwing(new LivyUserException("livy.unexpected_error"));
}
if (System.currentTimeMillis() > stopPolling || statement.getState().equals(StatementState.available)) {
break;
}
logger.trace("Statement was not ready, polling now with attempt '{}'", pollCount++);
// statement not ready, wait for some time...
try {
Thread.sleep(livyProperties.getPollingInterval());
} catch (InterruptedException e) {
logger.error("Thread interrupted while polling Livy", e);
}
} while (true);
logger.debug("exit DefaultLivyClient poll statement in '{}' millis, after '{}' attempts ", System.currentTimeMillis() - startMillis, pollCount);
return logger.exit(statement);
}
Aggregations