use of datawave.webservice.query.exception.BadRequestQueryException in project datawave by NationalSecurityAgency.
the class QueryExecutorBean method updateQuery.
private void updateQuery(GenericResponse<String> response, RunningQuery runningQuery, String queryLogicName, String query, Date beginDate, Date endDate, String queryAuthorizations, Date expirationDate, Integer pagesize, Integer pageTimeout, Long maxResultsOverride, QueryPersistence persistenceMode, String parameters) throws Exception {
// Find out who/what called this method
Principal p = ctx.getCallerPrincipal();
String userid = p.getName();
Collection<Collection<String>> cbAuths = new HashSet<>();
if (p instanceof DatawavePrincipal) {
DatawavePrincipal dp = (DatawavePrincipal) p;
userid = dp.getShortName();
cbAuths.addAll(dp.getAuthorizations());
}
log.trace(userid + " has authorizations " + cbAuths);
Query q = runningQuery.getSettings();
// validate persistence mode first
if (persistenceMode != null) {
switch(persistenceMode) {
case PERSISTENT:
if (q.getQueryName() == null)
throw new BadRequestQueryException(DatawaveErrorCode.QUERY_NAME_REQUIRED);
break;
case TRANSIENT:
break;
default:
throw new BadRequestQueryException(DatawaveErrorCode.UNKNOWN_PERSISTENCE_MODE, MessageFormat.format("Mode = {0}", persistenceMode));
}
}
// test for any auditable updates
if (query != null || beginDate != null || endDate != null || queryAuthorizations != null) {
// must clone/audit attempt first
Query duplicate = q.duplicate(q.getQueryName());
duplicate.setId(q.getId());
updateQueryParams(duplicate, queryLogicName, query, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, maxResultsOverride, parameters);
// Fire off an audit prior to updating
AuditType auditType = runningQuery.getLogic().getAuditType(runningQuery.getSettings());
if (!auditType.equals(AuditType.NONE)) {
try {
MultivaluedMap<String, String> queryParameters = new MultivaluedMapImpl<>();
queryParameters.putAll(duplicate.toMap());
// if the user didn't set an audit id, use the query id
if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
queryParameters.putSingle(AuditParameters.AUDIT_ID, q.getId().toString());
}
auditor.audit(queryParameters);
} catch (IllegalArgumentException e) {
log.error("Error validating audit parameters", e);
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
response.addException(qe);
throw new BadRequestException(qe, response);
} catch (Exception e) {
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e);
log.error(qe, e);
response.addException(qe.getBottomQueryException());
throw e;
}
}
}
// update the actual running query
updateQueryParams(q, queryLogicName, query, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, maxResultsOverride, parameters);
// update the persistenceMode post audit
if (persistenceMode != null) {
switch(persistenceMode) {
case PERSISTENT:
persister.update(q);
break;
case TRANSIENT:
persister.remove(q);
break;
}
}
// Put in the cache by id
queryCache.put(q.getId().toString(), runningQuery);
}
use of datawave.webservice.query.exception.BadRequestQueryException in project datawave by NationalSecurityAgency.
the class QueryExecutorBean method planQuery.
/**
* @param queryLogicName
* @param queryParameters
* @return
*/
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@Path("/{logicName}/plan")
@Interceptors({ RequiredInterceptor.class, ResponseInterceptor.class })
@Timed(name = "dw.query.planQuery", absolute = true)
public GenericResponse<String> planQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, MultivaluedMap<String, String> queryParameters) {
QueryData qd = validateQuery(queryLogicName, queryParameters, null);
GenericResponse<String> response = new GenericResponse<>();
Query q = null;
Connector connection = null;
AccumuloConnectionFactory.Priority priority;
try {
// Default hasResults to true.
response.setHasResults(true);
// by default we will expand the fields but not the values.
boolean expandFields = true;
boolean expandValues = false;
if (queryParameters.containsKey(EXPAND_FIELDS)) {
expandFields = Boolean.valueOf(queryParameters.getFirst(EXPAND_FIELDS));
}
if (queryParameters.containsKey(EXPAND_VALUES)) {
expandValues = Boolean.valueOf(queryParameters.getFirst(EXPAND_VALUES));
}
AuditType auditType = qd.logic.getAuditType(null);
try {
MultivaluedMap<String, String> optionalQueryParameters = new MultivaluedMapImpl<>();
optionalQueryParameters.putAll(qp.getUnknownParameters(queryParameters));
q = persister.create(qd.userDn, qd.dnList, marking, queryLogicName, qp, optionalQueryParameters);
auditType = qd.logic.getAuditType(q);
} finally {
queryParameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name());
// on audit if needed, and we are using the index to expand the values
if (expandValues && !auditType.equals(AuditType.NONE)) {
// audit the query before its executed.
try {
try {
List<String> selectors = qd.logic.getSelectors(q);
if (selectors != null && !selectors.isEmpty()) {
queryParameters.put(PrivateAuditConstants.SELECTORS, selectors);
}
} catch (Exception e) {
log.error("Error accessing query selector", e);
}
// if the user didn't set an audit id, use the query id
if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
queryParameters.putSingle(AuditParameters.AUDIT_ID, q.getId().toString());
}
auditor.audit(queryParameters);
} catch (IllegalArgumentException e) {
log.error("Error validating audit parameters", e);
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
response.addException(qe);
throw new BadRequestException(qe, response);
} catch (Exception e) {
log.error("Error auditing query", e);
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e);
response.addException(qe);
throw qe;
}
}
}
priority = qd.logic.getConnectionPriority();
Map<String, String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
addQueryToTrackingMap(trackingMap, q);
accumuloConnectionRequestBean.requestBegin(q.getId().toString());
try {
connection = connectionFactory.getConnection(qd.logic.getConnPoolName(), priority, trackingMap);
} finally {
accumuloConnectionRequestBean.requestEnd(q.getId().toString());
}
Set<Authorizations> calculatedAuths = AuthorizationsUtil.getDowngradedAuthorizations(qp.getAuths(), qd.p);
String plan = qd.logic.getPlan(connection, q, calculatedAuths, expandFields, expandValues);
response.setResult(plan);
return response;
} catch (Throwable t) {
response.setHasResults(false);
/*
* Allow web services to throw their own WebApplicationExceptions
*/
if (t instanceof Error && !(t instanceof TokenMgrError)) {
log.error(t.getMessage(), t);
throw (Error) t;
} else if (t instanceof WebApplicationException) {
log.error(t.getMessage(), t);
throw ((WebApplicationException) t);
} else {
log.error(t.getMessage(), t);
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_PLAN_ERROR, t);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode);
}
} finally {
if (connection != null) {
try {
connectionFactory.returnConnection(connection);
} catch (Exception e) {
log.error("Failed to close connection for " + q.getId(), e);
}
}
// close the logic on exception
try {
if (null != qd.logic) {
qd.logic.close();
}
} catch (Exception e) {
log.error("Exception occured while closing query logic; may be innocuous if scanners were running.", e);
}
if (null != connection) {
try {
connectionFactory.returnConnection(connection);
} catch (Exception e) {
log.error("Error returning connection on failed create", e);
}
}
}
}
use of datawave.webservice.query.exception.BadRequestQueryException in project datawave by NationalSecurityAgency.
the class QueryExecutorBean method createQuery.
/**
* @param queryLogicName
* @param queryParameters
* @return
*/
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@Path("/{logicName}/create")
@GZIP
@GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/")
@EnrichQueryMetrics(methodType = MethodType.CREATE)
@Interceptors({ RequiredInterceptor.class, ResponseInterceptor.class })
@Timed(name = "dw.query.createQuery", absolute = true)
public GenericResponse<String> createQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, MultivaluedMap<String, String> queryParameters, @Context HttpHeaders httpHeaders) {
CreateQuerySessionIDFilter.QUERY_ID.set(null);
QueryData qd = validateQuery(queryLogicName, queryParameters, httpHeaders);
GenericResponse<String> response = new GenericResponse<>();
Query q = null;
Connector connection = null;
AccumuloConnectionFactory.Priority priority;
Span createSpan = null;
RunningQuery rq = null;
try {
// Default hasResults to true. If a query logic is actually able to set this value,
// then their value will overwrite this one. Otherwise, we return true so that
// callers know they have to call next (even though next may not return results).
response.setHasResults(true);
AuditType auditType = qd.logic.getAuditType(null);
try {
MultivaluedMap<String, String> optionalQueryParameters = new MultivaluedMapImpl<>();
optionalQueryParameters.putAll(qp.getUnknownParameters(queryParameters));
q = persister.create(qd.userDn, qd.dnList, marking, queryLogicName, qp, optionalQueryParameters);
auditType = qd.logic.getAuditType(q);
} finally {
queryParameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name());
if (!auditType.equals(AuditType.NONE)) {
// audit the query before its executed.
try {
try {
List<String> selectors = qd.logic.getSelectors(q);
if (selectors != null && !selectors.isEmpty()) {
queryParameters.put(PrivateAuditConstants.SELECTORS, selectors);
}
} catch (Exception e) {
log.error("Error accessing query selector", e);
}
// if the user didn't set an audit id, use the query id
if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
queryParameters.putSingle(AuditParameters.AUDIT_ID, q.getId().toString());
}
auditor.audit(queryParameters);
} catch (IllegalArgumentException e) {
log.error("Error validating audit parameters", e);
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
response.addException(qe);
throw new BadRequestException(qe, response);
} catch (Exception e) {
log.error("Error auditing query", e);
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e);
response.addException(qe);
throw qe;
}
}
}
priority = qd.logic.getConnectionPriority();
Map<String, String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
addQueryToTrackingMap(trackingMap, q);
accumuloConnectionRequestBean.requestBegin(q.getId().toString());
try {
connection = connectionFactory.getConnection(qd.logic.getConnPoolName(), priority, trackingMap);
} finally {
accumuloConnectionRequestBean.requestEnd(q.getId().toString());
}
// If we're supposed to trace this query, then turn tracing on and set information about the query
// onto the span so that it is saved in the trace table.
TInfo traceInfo = null;
boolean shouldTraceQuery = shouldTraceQuery(qp.getQuery(), qd.userid, qp.isTrace());
if (shouldTraceQuery) {
Span span = Trace.on("query:" + q.getId());
log.debug("Tracing query " + q.getId() + " [" + qp.getQuery() + "] on trace ID " + Long.toHexString(span.traceId()));
for (Entry<String, List<String>> param : queryParameters.entrySet()) {
span.data(param.getKey(), param.getValue().get(0));
}
traceInfo = Tracer.traceInfo();
createSpan = Trace.start("query:create");
}
// hold on to a reference of the query logic so we cancel it if need be.
qlCache.add(q.getId().toString(), qd.userid, qd.logic, connection);
rq = new RunningQuery(metrics, null, priority, qd.logic, q, qp.getAuths(), qd.p, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
rq.setActiveCall(true);
rq.setTraceInfo(traceInfo);
rq.getMetric().setProxyServers(qd.proxyServers);
rq.setConnection(connection);
// Put in the cache by id. Don't put the cache in by name because multiple users may use the same name
// and only the last one will be in the cache.
queryCache.put(q.getId().toString(), rq);
response.setResult(q.getId().toString());
rq.setActiveCall(false);
CreateQuerySessionIDFilter.QUERY_ID.set(q.getId().toString());
return response;
} catch (Throwable t) {
response.setHasResults(false);
if (rq != null) {
rq.getMetric().setError(t);
}
// close the logic on exception
try {
if (null != qd.logic) {
qd.logic.close();
}
} catch (Exception e) {
log.error("Exception occured while closing query logic; may be innocuous if scanners were running.", e);
}
if (null != connection) {
try {
connectionFactory.returnConnection(connection);
} catch (Exception e) {
log.error("Error returning connection on failed create", e);
}
}
try {
if (null != q)
persister.remove(q);
} catch (Exception e) {
response.addException(new QueryException(DatawaveErrorCode.DEPERSIST_ERROR, e).getBottomQueryException());
}
/*
* Allow web services to throw their own WebApplicationExceptions
*/
if (t instanceof Error && !(t instanceof TokenMgrError)) {
log.error(t.getMessage(), t);
throw (Error) t;
} else if (t instanceof WebApplicationException) {
log.error(t.getMessage(), t);
throw ((WebApplicationException) t);
} else if (t instanceof InterruptedException) {
if (rq != null) {
rq.getMetric().setLifecycle(QueryMetric.Lifecycle.CANCELLED);
}
log.info("Query " + q.getId() + " canceled on request");
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_CANCELED, t);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode);
} else {
log.error(t.getMessage(), t);
QueryException qe = new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, t);
response.addException(qe.getBottomQueryException());
int statusCode = qe.getBottomQueryException().getStatusCode();
throw new DatawaveWebApplicationException(qe, response, statusCode);
}
} finally {
if (createSpan != null) {
createSpan.stop();
// TODO: not sure this makes any sense anymore in Accumulo 1.8.1
// Stop the main query span since we're done working with it on this thread.
// We'll continue it later.
// createSpan.parent().stop();
}
if (null != q) {
// - Remove the logic from the cache
qlCache.poll(q.getId().toString());
}
}
}
use of datawave.webservice.query.exception.BadRequestQueryException in project datawave by NationalSecurityAgency.
the class MapReduceBean method ooziesubmit.
/**
* Execute a Oozie workflow with the given workFlow name and runtime parameters
*
* @param queryParameters
* @return
*/
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@javax.ws.rs.Path("/ooziesubmit")
@GZIP
public GenericResponse<String> ooziesubmit(MultivaluedMap<String, String> queryParameters) {
GenericResponse<String> response = new GenericResponse<>();
String workFlow = queryParameters.getFirst(OozieJobConstants.WORKFLOW_PARAM);
if (StringUtils.isBlank(workFlow)) {
throw new BadRequestException(new IllegalArgumentException(OozieJobConstants.WORKFLOW_PARAM + " parameter missing"), response);
}
String parameters = queryParameters.getFirst(OozieJobConstants.PARAMETERS);
// Find out who/what called this method
Principal p = ctx.getCallerPrincipal();
String sid = null;
String userDn = p.getName();
DatawavePrincipal datawavePrincipal = null;
if (p instanceof DatawavePrincipal) {
datawavePrincipal = (DatawavePrincipal) p;
sid = datawavePrincipal.getShortName();
} else {
QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_PRINCIPAL_ERROR, MessageFormat.format("Class: {0}", p.getClass().getName()));
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
}
OozieJobConfiguration job;
try {
MapReduceJobConfiguration mrConfig = this.mapReduceConfiguration.getConfiguration(workFlow);
if (mrConfig instanceof OozieJobConfiguration) {
job = (OozieJobConfiguration) mrConfig;
} else {
throw new IllegalArgumentException(workFlow + " not an Oozie job configuration");
}
} catch (IllegalArgumentException e) {
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.JOB_CONFIGURATION_ERROR, e);
response.addException(qe);
throw new BadRequestException(qe, response);
}
if (job instanceof NeedCallerDetails) {
((NeedCallerDetails) job).setUserSid(sid);
((NeedCallerDetails) job).setPrincipal(p);
}
// Ensure that the user has the required roles and has passed the required auths
if (null != job.getRequiredRoles() || null != job.getRequiredAuths()) {
try {
canRunJob(datawavePrincipal, queryParameters, job.getRequiredRoles(), job.getRequiredAuths());
} catch (UnauthorizedQueryException qe) {
// user does not have all of the required roles or did not pass the required auths
response.addException(qe);
throw new UnauthorizedException(qe, response);
}
}
String id = sid + "_" + UUID.randomUUID();
OozieClient oozieClient = null;
Properties oozieConf = null;
try {
oozieClient = new OozieClient((String) job.getJobConfigurationProperties().get(OozieJobConstants.OOZIE_CLIENT_PROP));
oozieConf = oozieClient.createConfiguration();
job.initializeOozieConfiguration(id, oozieConf, queryParameters);
job.validateWorkflowParameter(oozieConf, mapReduceConfiguration);
} catch (QueryException qe) {
log.error(qe.getMessage(), qe);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
} catch (Exception e) {
log.error(e.getMessage(), e);
response.addException(new QueryException(e.getMessage(), e));
throw new DatawaveWebApplicationException(e, response);
} finally {
// audit query here
Auditor.AuditType auditType = job.getAuditType();
log.trace("Audit type is: " + auditType.name());
if (!auditType.equals(Auditor.AuditType.NONE)) {
try {
marking.validate(queryParameters);
PrivateAuditConstants.stripPrivateParameters(queryParameters);
queryParameters.putSingle(PrivateAuditConstants.USER_DN, userDn);
queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString());
queryParameters.putSingle(PrivateAuditConstants.AUDIT_TYPE, auditType.name());
List<String> selectors = job.getSelectors(queryParameters, oozieConf);
if (selectors != null && !selectors.isEmpty()) {
queryParameters.put(PrivateAuditConstants.SELECTORS, selectors);
}
// if the user didn't set an audit id, use the query id
if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
queryParameters.putSingle(AuditParameters.AUDIT_ID, id);
}
auditor.audit(queryParameters);
} catch (IllegalArgumentException e) {
log.error("Error validating audit parameters", e);
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
response.addException(qe);
throw new BadRequestException(qe, response);
} catch (Exception e) {
log.error("Error auditing query", e);
response.addMessage("Error auditing query - " + e.getMessage());
throw new BadRequestException(e, response);
}
}
}
// Submit the Oozie workflow.
try {
String jobID = null;
try {
jobID = oozieClient.run(oozieConf);
} catch (Exception e) {
throw new QueryException(DatawaveErrorCode.OOZIE_JOB_START_ERROR, e);
}
try {
String jobResultstDir = oozieConf.getProperty(OozieJobConstants.OUT_DIR_PROP) + "/" + id;
response.setResult(id);
Path baseDir = new Path(this.mapReduceConfiguration.getMapReduceBaseDirectory());
// Create a directory path for this job
Path jobDir = new Path(baseDir, id);
mapReduceState.create(id, job.getHdfsUri(), job.getJobTracker(), jobDir.toString(), jobID, jobResultstDir, parameters, workFlow);
} catch (Exception e) {
QueryException qe = new QueryException(DatawaveErrorCode.MAPREDUCE_STATE_PERSISTENCE_ERROR, e);
response.addException(qe.getBottomQueryException());
try {
oozieClient.kill(jobID);
// if we successfully kill the job, throw the original exception
throw qe;
} catch (Exception e2) {
// throw the exception from killing the job
throw new QueryException(DatawaveErrorCode.MAPREDUCE_JOB_KILL_ERROR, e2);
}
}
} catch (QueryException qe) {
log.error(qe.getMessage(), qe);
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
} catch (Exception e) {
log.error(e.getMessage(), e);
QueryException qe = new QueryException(DatawaveErrorCode.UNKNOWN_SERVER_ERROR, e.getMessage());
response.addException(qe);
throw new DatawaveWebApplicationException(qe, response);
}
return response;
}
use of datawave.webservice.query.exception.BadRequestQueryException in project datawave by NationalSecurityAgency.
the class EdgeTableRangeBuildingVisitor method visit.
/**
* Used only for the Edge'Source'QueryLogic Basically function support was an afterthought only to support old code in the existing Edge'Source'QueryLogic A
* list with a single Identity context is returned in the following format: Identity = FUNCTION Opperator = FUNCTION LITER = "{function string}" //eg
* source.has_all(SINK, "t1", "t2", ...)
*/
public Object visit(ASTFunctionNode node, Object data) {
if (node.jjtGetParent() instanceof ASTJexlScript) {
// then we're the only node in here, we need to come up with ranges from the function itself, or fail the query
throw new IllegalArgumentException("Cannot perform has_all without specifying a pattern for the edge 'SOURCE'.");
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < node.jjtGetNumChildren(); i++) {
if (0 == i) {
sb.append(node.jjtGetChild(i).image);
} else if (1 == i) {
if (!allowedFunctions.contains(node.jjtGetChild(i).image.toLowerCase())) {
BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.FUNCTION_NOT_FOUND, node.jjtGetChild(i).image + " not supported function for EdgeQuery");
throw new UnsupportedOperationException(qe);
}
sb.append(":");
sb.append(node.jjtGetChild(i).image);
} else if (2 == i) {
sb.append("(");
sb.append(getLiteral(node.jjtGetChild(i)));
} else if (2 < i) {
sb.append(", ");
sb.append("'" + getLiteral(node.jjtGetChild(i)).toLowerCase() + "'");
}
}
sb.append(")");
List<IdentityContext> contexts = new ArrayList<>();
IdentityContext iContext = new IdentityContext(FUNCTION, sb.toString(), FUNCTION);
contexts.add(iContext);
return contexts;
}
Aggregations