use of com.thinkbiganalytics.kylo.spark.exceptions.LivyUserException in project kylo by Teradata.
the class LivyRestModelTransformer method toTransformResponse.
public static TransformResponse toTransformResponse(Statement statement, String transformId) {
TransformResponse transformResponse = prepTransformResponse(statement, transformId);
if (transformResponse.getStatus() == TransformResponse.Status.SUCCESS) {
String code = statement.getCode().trim();
if (code.endsWith("dfResultsAsJson")) {
transformResponse.setResults(toTransformQueryResultWithSchema(transformResponse, statement.getOutput()));
} else if (code.endsWith("dfProf")) {
List<OutputRow> rows = toTransformResponseProfileStats(statement.getOutput());
transformResponse.setProfile(toTransformResponseProfileStats(statement.getOutput()));
transformResponse.setActualCols(1);
Integer actualRows = rows.stream().filter(metric -> metric.getMetricType().equals(MetricType.TOTAL_COUNT.toString())).map(metric -> Integer.valueOf(metric.getMetricValue())).findFirst().orElse(1);
transformResponse.setActualRows(actualRows);
transformResponse.setResults(emptyResult());
} else if (code.endsWith("transformAsStr")) {
/* expects that 'statement' contains a payload of TransformResponse in JSON format */
TransformResponse tr = serializeStatementOutputResponse(checkCodeWasWellFormed(statement.getOutput()), TransformResponse.class);
statementIdCache.put(tr.getTable(), statement.getId());
return tr;
} else {
logger.error("Exception Processing Result: ", new LivyCodeException("Unsupported result type requested of Livy. Results not recognized"));
throw new LivyUserException("livy.unsupported_result_type");
}
// end if
}
return transformResponse;
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyUserException in project kylo by Teradata.
the class LivyRestModelTransformer method convertDataFrameDataType.
private static String convertDataFrameDataType(JsonNode dataType) {
if (dataType.isObject()) {
String type = dataType.get("type").asText();
if (type.equals("udt")) {
if (dataType.get("class").asText().equals("org.apache.spark.mllib.linalg.VectorUDT")) {
// TODO: null check
ArrayNode fields = (ArrayNode) dataType.get("sqlType").get("fields");
Iterator<JsonNode> fieldsIter = fields.elements();
while (fieldsIter.hasNext()) {
ObjectNode fieldDescriptors = (ObjectNode) fieldsIter.next();
if (fieldDescriptors.get("name").asText().equals("values")) {
ObjectNode fdObj = (ObjectNode) fieldDescriptors.get("type");
return new StringBuilder(fdObj.get("type").asText()).append("<").append(fdObj.get("elementType").asText()).append(">").toString();
}
}
return "Unknown UDT";
} else {
if (dataType.get("class") != null) {
logger.error("UDT error encountered", new LivyDeserializationException("don't know how to deserialize UDT types for class = " + dataType.get("class").asText()));
} else {
logger.error("UDT error encountered", new LivyDeserializationException("don't know how to deserialize UDT type of unspecified class"));
}
// end if
throw new LivyUserException("livy.unexpected_error");
}
// end if
} else if (type.equals("map")) {
return new StringBuilder(dataType.get("type").asText()).append("<").append(dataType.get("keyType").asText()).append(",").append(dataType.get("valueType").asText()).append(">").toString();
} else if (type.equals("struct")) {
ArrayNode fields = (ArrayNode) dataType.get("fields");
Iterator<JsonNode> nodes = fields.elements();
StringBuilder sb = new StringBuilder("struct<");
// assumes min of 1 field in struct
while (nodes.hasNext()) {
ObjectNode node = (ObjectNode) nodes.next();
String sfName = node.get("name").asText();
String sfType = node.get("type").asText();
sb.append(sfName).append(":").append(sfType).append(",");
}
sb.deleteCharAt(sb.length() - 1);
return sb.toString();
} else {
// can there be other types?
return "Unknown Type";
}
// end if
} else {
return dataType.asText();
}
// end if
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyUserException in project kylo by Teradata.
the class LivyRestModelTransformer method checkCodeWasWellFormed.
private static StatementOutputResponse checkCodeWasWellFormed(StatementOutputResponse statementOutputResponse) {
if (statementOutputResponse != null && statementOutputResponse.getStatus() != StatementOutputStatus.ok) {
String msg = String.format("Malformed code sent to Livy. ErrorType='%s', Error='%s', Traceback='%s'", statementOutputResponse.getEname(), statementOutputResponse.getEvalue(), statementOutputResponse.getTraceback());
logger.error("Exception Processing Query: ", new LivyCodeException(msg));
throw new LivyUserException("livy.syntax");
}
return statementOutputResponse;
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyUserException in project kylo by Teradata.
the class LivyRestModelTransformer method serializeStatementOutputResponse.
private static <T extends Object> T serializeStatementOutputResponse(StatementOutputResponse sor, Class<T> clazz) {
String errMsg = String.format("Unable to deserialize JSON returned from Livy into class '%s'", clazz.getSimpleName());
JsonNode data = sor.getData();
if (data != null) {
JsonNode json = data.get("application/json");
String jsonString = json.asText();
try {
return mapper.readValue(jsonString, clazz);
} catch (IOException e) {
logger.error("Deserialization error occured", new LivyDeserializationException(errMsg));
}
// end try/catch
} else {
logger.error("Deserialization error occured", new LivyDeserializationException(errMsg));
}
throw new LivyUserException("livy.unexpected_error");
}
use of com.thinkbiganalytics.kylo.spark.exceptions.LivyUserException in project kylo by Teradata.
the class SparkLivyRestClient method getTransformResult.
@Nonnull
@Override
public Optional<TransformResponse> getTransformResult(@Nonnull SparkShellProcess process, @Nonnull String transformId) {
logger.entry(process, transformId);
Validate.isInstanceOf(SparkLivyProcess.class, process, "SparkLivyRestClient.getTransformResult called on non Livy Process");
SparkLivyProcess sparkLivyProcess = (SparkLivyProcess) process;
JerseyRestClient client = sparkLivyProcessManager.getClient(process);
Integer stmtId = sparkLivyProcessManager.getStatementId(transformId);
Statement statement = livyClient.getStatement(client, sparkLivyProcess, stmtId);
sparkLivyProcessManager.setStatementId(transformId, statement.getId());
TransformResponse response = LivyRestModelTransformer.toTransformResponse(statement, transformId);
if (statement.getState() == StatementState.available && response.getStatus() == TransformResponse.Status.PENDING) {
// TODO:: change this so that if transformId is a livyQueryID it knows what to do. similar to saveResponse. A good first stab at this is at KYLO-2639-withLivyId
// The result came back from Livy, but the transform result still needs to be fetched
String script = scriptGenerator.script("getTransform", ScalaScriptUtils.scalaStr(transformId));
statement = submitCode(client, script, process);
// associate transformId to this new query and get results on subsequent call
sparkLivyProcessManager.setStatementId(transformId, statement.getId());
} else if (response.getStatus() == TransformResponse.Status.ERROR) {
logger.error(String.format("Unexpected error found in transform response:\n%s", response.getMessage()));
throw new LivyUserException("livy.transform_error");
}
// end if
return logger.exit(Optional.of(response));
}
Aggregations