Examples with RunningQueryTimingImpl datawave.webservice.query.cache.RunningQueryTimingImpl used on opensource projects

Example 1 with RunningQueryTimingImpl

use of datawave.webservice.query.cache.RunningQueryTimingImpl in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method getQueryById.

private RunningQuery getQueryById(String id, Principal principal) throws Exception {
    // Find out who/what called this method
    String userid = principal.getName();
    if (principal instanceof DatawavePrincipal) {
        DatawavePrincipal dp = (DatawavePrincipal) principal;
        userid = dp.getShortName();
    }
    log.trace(userid + " has authorizations " + ((principal instanceof DatawavePrincipal) ? ((DatawavePrincipal) principal).getAuthorizations() : ""));
    RunningQuery query = queryCache.get(id);
    if (null == query) {
        log.info("Query not found in cache, retrieving from accumulo");
        List<Query> queries = persister.findById(id);
        if (null == queries || queries.isEmpty())
            throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH);
        if (queries.size() > 1)
            throw new NotFoundQueryException(DatawaveErrorCode.TOO_MANY_QUERY_OBJECT_MATCHES);
        else {
            Query q = queries.get(0);
            // will throw IllegalArgumentException if not defined
            QueryLogic<?> logic = queryLogicFactory.getQueryLogic(q.getQueryLogicName(), principal);
            AccumuloConnectionFactory.Priority priority = logic.getConnectionPriority();
            query = new RunningQuery(metrics, null, priority, logic, q, q.getQueryAuthorizations(), principal, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
            // Put in the cache by id and name, we will have two copies that reference the same object
            queryCache.put(q.getId().toString(), query);
        }
    } else {
        // Check to make sure that this query belongs to current user.
        if (!query.getSettings().getOwner().equals(userid)) {
            throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userid, query.getSettings().getOwner()));
        }
    }
    return query;
}

Example 2 with RunningQueryTimingImpl

use of datawave.webservice.query.cache.RunningQueryTimingImpl in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method defineQuery.

/**
 * @param queryLogicName
 * @param queryParameters
 * @return
 */
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@Path("/{logicName}/define")
@GZIP
@GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/")
@EnrichQueryMetrics(methodType = MethodType.CREATE)
@Interceptors({ RequiredInterceptor.class, ResponseInterceptor.class })
@Timed(name = "dw.query.defineQuery", absolute = true)
public GenericResponse<String> defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, MultivaluedMap<String, String> queryParameters, @Context HttpHeaders httpHeaders) {
    CreateQuerySessionIDFilter.QUERY_ID.set(null);
    QueryData qd = validateQuery(queryLogicName, queryParameters, httpHeaders);
    GenericResponse<String> response = new GenericResponse<>();
    // We need to put a disconnected RunningQuery instance into the cache. Otherwise TRANSIENT queries
    // will not exist when reset is called.
    Span defineSpan = null;
    RunningQuery rq;
    try {
        MultivaluedMap<String, String> optionalQueryParameters = new MultivaluedMapImpl<>();
        optionalQueryParameters.putAll(qp.getUnknownParameters(queryParameters));
        Query q = persister.create(qd.userDn, qd.dnList, marking, queryLogicName, qp, optionalQueryParameters);
        response.setResult(q.getId().toString());
        // If we're supposed to trace this query, then turn tracing on and set information about the query
        // onto the span so that it is saved in the trace table.
        TInfo traceInfo = null;
        boolean shouldTraceQuery = shouldTraceQuery(qp.getQuery(), qd.userid, false);
        if (shouldTraceQuery) {
            Span span = Trace.on("query:" + q.getId());
            log.debug("Tracing query " + q.getId() + " [" + qp.getQuery() + "] on trace ID " + Long.toHexString(span.traceId()));
            for (Entry<String, List<String>> param : queryParameters.entrySet()) {
                span.data(param.getKey(), param.getValue().get(0));
            }
            traceInfo = Tracer.traceInfo();
            defineSpan = Trace.start("query:define");
        }
        AccumuloConnectionFactory.Priority priority = qd.logic.getConnectionPriority();
        rq = new RunningQuery(metrics, null, priority, qd.logic, q, qp.getAuths(), qd.p, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
        rq.setActiveCall(true);
        rq.getMetric().setProxyServers(qd.proxyServers);
        rq.setTraceInfo(traceInfo);
        queryCache.put(q.getId().toString(), rq);
        rq.setActiveCall(false);
        CreateQuerySessionIDFilter.QUERY_ID.set(q.getId().toString());
        return response;
    } catch (DatawaveWebApplicationException e) {
        throw e;
    } catch (Exception e) {
        log.error("Error accessing optional query parameters", e);
        QueryException qe = new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e);
        response.addException(qe.getBottomQueryException());
        int statusCode = qe.getBottomQueryException().getStatusCode();
        throw new DatawaveWebApplicationException(qe, response, statusCode);
    } finally {
        if (null != defineSpan) {
            // couple milliseconds just to ensure we get something saved.
            try {
                Thread.sleep(2);
            } catch (InterruptedException e) {
            // ignore
            }
            defineSpan.stop();
        // TODO: not sure this makes any sense anymore in Accumulo 1.8.1
        // if (null != defineSpan.parent()) {
        // // Stop the main query span since we're done working with it on this thread.
        // // We'll continue it later.
        // defineSpan.parent().stop();
        // }
        }
    }
}

Example 3 with RunningQueryTimingImpl

use of datawave.webservice.query.cache.RunningQueryTimingImpl in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method getQueryByName.

private List<RunningQuery> getQueryByName(String name) throws Exception {
    // Find out who/what called this method
    Principal p = ctx.getCallerPrincipal();
    String userid = p.getName();
    if (p instanceof DatawavePrincipal) {
        DatawavePrincipal dp = (DatawavePrincipal) p;
        userid = dp.getShortName();
    }
    log.trace(userid + " has authorizations " + ((p instanceof DatawavePrincipal) ? ((DatawavePrincipal) p).getAuthorizations() : ""));
    List<RunningQuery> results = new ArrayList<>();
    List<Query> queries = persister.findByName(name);
    if (null == queries)
        throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH);
    for (Query q : queries) {
        // never get back other people queries. Leaving for now just in case the persister changes.
        if (!q.getOwner().equals(userid)) {
            throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userid, q.getOwner()));
        }
        // will throw IllegalArgumentException if not defined
        QueryLogic<?> logic = queryLogicFactory.getQueryLogic(q.getQueryLogicName(), p);
        AccumuloConnectionFactory.Priority priority = logic.getConnectionPriority();
        RunningQuery query = new RunningQuery(metrics, null, priority, logic, q, q.getQueryAuthorizations(), p, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
        results.add(query);
        // Put in the cache by id if its not already in the cache.
        if (!queryCache.containsKey(q.getId().toString()))
            queryCache.put(q.getId().toString(), query);
    }
    return results;
}

Example 4 with RunningQueryTimingImpl

use of datawave.webservice.query.cache.RunningQueryTimingImpl in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method createQuery.

/**
 * @param queryLogicName
 * @param queryParameters
 * @return
 */
@POST
@Produces({ "application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff" })
@Path("/{logicName}/create")
@GZIP
@GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/")
@EnrichQueryMetrics(methodType = MethodType.CREATE)
@Interceptors({ RequiredInterceptor.class, ResponseInterceptor.class })
@Timed(name = "dw.query.createQuery", absolute = true)
public GenericResponse<String> createQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, MultivaluedMap<String, String> queryParameters, @Context HttpHeaders httpHeaders) {
    CreateQuerySessionIDFilter.QUERY_ID.set(null);
    QueryData qd = validateQuery(queryLogicName, queryParameters, httpHeaders);
    GenericResponse<String> response = new GenericResponse<>();
    Query q = null;
    Connector connection = null;
    AccumuloConnectionFactory.Priority priority;
    Span createSpan = null;
    RunningQuery rq = null;
    try {
        // Default hasResults to true. If a query logic is actually able to set this value,
        // then their value will overwrite this one. Otherwise, we return true so that
        // callers know they have to call next (even though next may not return results).
        response.setHasResults(true);
        AuditType auditType = qd.logic.getAuditType(null);
        try {
            MultivaluedMap<String, String> optionalQueryParameters = new MultivaluedMapImpl<>();
            optionalQueryParameters.putAll(qp.getUnknownParameters(queryParameters));
            q = persister.create(qd.userDn, qd.dnList, marking, queryLogicName, qp, optionalQueryParameters);
            auditType = qd.logic.getAuditType(q);
        } finally {
            queryParameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name());
            if (!auditType.equals(AuditType.NONE)) {
                // audit the query before its executed.
                try {
                    try {
                        List<String> selectors = qd.logic.getSelectors(q);
                        if (selectors != null && !selectors.isEmpty()) {
                            queryParameters.put(PrivateAuditConstants.SELECTORS, selectors);
                        }
                    } catch (Exception e) {
                        log.error("Error accessing query selector", e);
                    }
                    // if the user didn't set an audit id, use the query id
                    if (!queryParameters.containsKey(AuditParameters.AUDIT_ID)) {
                        queryParameters.putSingle(AuditParameters.AUDIT_ID, q.getId().toString());
                    }
                    auditor.audit(queryParameters);
                } catch (IllegalArgumentException e) {
                    log.error("Error validating audit parameters", e);
                    BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e);
                    response.addException(qe);
                    throw new BadRequestException(qe, response);
                } catch (Exception e) {
                    log.error("Error auditing query", e);
                    QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e);
                    response.addException(qe);
                    throw qe;
                }
            }
        }
        priority = qd.logic.getConnectionPriority();
        Map<String, String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
        addQueryToTrackingMap(trackingMap, q);
        accumuloConnectionRequestBean.requestBegin(q.getId().toString());
        try {
            connection = connectionFactory.getConnection(qd.logic.getConnPoolName(), priority, trackingMap);
        } finally {
            accumuloConnectionRequestBean.requestEnd(q.getId().toString());
        }
        // If we're supposed to trace this query, then turn tracing on and set information about the query
        // onto the span so that it is saved in the trace table.
        TInfo traceInfo = null;
        boolean shouldTraceQuery = shouldTraceQuery(qp.getQuery(), qd.userid, qp.isTrace());
        if (shouldTraceQuery) {
            Span span = Trace.on("query:" + q.getId());
            log.debug("Tracing query " + q.getId() + " [" + qp.getQuery() + "] on trace ID " + Long.toHexString(span.traceId()));
            for (Entry<String, List<String>> param : queryParameters.entrySet()) {
                span.data(param.getKey(), param.getValue().get(0));
            }
            traceInfo = Tracer.traceInfo();
            createSpan = Trace.start("query:create");
        }
        // hold on to a reference of the query logic so we cancel it if need be.
        qlCache.add(q.getId().toString(), qd.userid, qd.logic, connection);
        rq = new RunningQuery(metrics, null, priority, qd.logic, q, qp.getAuths(), qd.p, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
        rq.setActiveCall(true);
        rq.setTraceInfo(traceInfo);
        rq.getMetric().setProxyServers(qd.proxyServers);
        rq.setConnection(connection);
        // Put in the cache by id. Don't put the cache in by name because multiple users may use the same name
        // and only the last one will be in the cache.
        queryCache.put(q.getId().toString(), rq);
        response.setResult(q.getId().toString());
        rq.setActiveCall(false);
        CreateQuerySessionIDFilter.QUERY_ID.set(q.getId().toString());
        return response;
    } catch (Throwable t) {
        response.setHasResults(false);
        if (rq != null) {
            rq.getMetric().setError(t);
        }
        // close the logic on exception
        try {
            if (null != qd.logic) {
                qd.logic.close();
            }
        } catch (Exception e) {
            log.error("Exception occured while closing query logic; may be innocuous if scanners were running.", e);
        }
        if (null != connection) {
            try {
                connectionFactory.returnConnection(connection);
            } catch (Exception e) {
                log.error("Error returning connection on failed create", e);
            }
        }
        try {
            if (null != q)
                persister.remove(q);
        } catch (Exception e) {
            response.addException(new QueryException(DatawaveErrorCode.DEPERSIST_ERROR, e).getBottomQueryException());
        }
        /*
             * Allow web services to throw their own WebApplicationExceptions
             */
        if (t instanceof Error && !(t instanceof TokenMgrError)) {
            log.error(t.getMessage(), t);
            throw (Error) t;
        } else if (t instanceof WebApplicationException) {
            log.error(t.getMessage(), t);
            throw ((WebApplicationException) t);
        } else if (t instanceof InterruptedException) {
            if (rq != null) {
                rq.getMetric().setLifecycle(QueryMetric.Lifecycle.CANCELLED);
            }
            log.info("Query " + q.getId() + " canceled on request");
            QueryException qe = new QueryException(DatawaveErrorCode.QUERY_CANCELED, t);
            response.addException(qe.getBottomQueryException());
            int statusCode = qe.getBottomQueryException().getStatusCode();
            throw new DatawaveWebApplicationException(qe, response, statusCode);
        } else {
            log.error(t.getMessage(), t);
            QueryException qe = new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, t);
            response.addException(qe.getBottomQueryException());
            int statusCode = qe.getBottomQueryException().getStatusCode();
            throw new DatawaveWebApplicationException(qe, response, statusCode);
        }
    } finally {
        if (createSpan != null) {
            createSpan.stop();
        // TODO: not sure this makes any sense anymore in Accumulo 1.8.1
        // Stop the main query span since we're done working with it on this thread.
        // We'll continue it later.
        // createSpan.parent().stop();
        }
        if (null != q) {
            // - Remove the logic from the cache
            qlCache.poll(q.getId().toString());
        }
    }
}

Example 5 with RunningQueryTimingImpl

use of datawave.webservice.query.cache.RunningQueryTimingImpl in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method adminGetQueryById.

private RunningQuery adminGetQueryById(String id) throws Exception {
    RunningQuery query = queryCache.get(id);
    if (query == null) {
        log.info("Query not found in cache, retrieving from accumulo");
        List<Query> queries = persister.adminFindById(id);
        if (queries == null || queries.isEmpty())
            throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH);
        if (queries.size() > 1)
            throw new NotFoundQueryException(DatawaveErrorCode.TOO_MANY_QUERY_OBJECT_MATCHES);
        Query q = queries.get(0);
        final String auths = q.getQueryAuthorizations();
        // will throw IllegalArgumentException if not defined
        final QueryLogic<?> logic = queryLogicFactory.getQueryLogic(q.getQueryLogicName(), ctx.getCallerPrincipal());
        final AccumuloConnectionFactory.Priority priority = logic.getConnectionPriority();
        query = RunningQuery.createQueryWithAuthorizations(metrics, null, priority, logic, q, auths, new RunningQueryTimingImpl(queryExpirationConf, qp.getPageTimeout()), this.executor, this.predictor, this.metricFactory);
        // Put in the cache by id and name, we will have two copies that reference the same object
        queryCache.put(q.getId().toString(), query);
    }
    return query;
}