Examples with Parameter datawave.webservice.query.QueryImpl.Parameter used on opensource projects

Example 11 with Parameter

use of datawave.webservice.query.QueryImpl.Parameter in project datawave by NationalSecurityAgency.

the class QueryUtilTest method testSerializationDeserialization.

@Test
public void testSerializationDeserialization() throws InvalidProtocolBufferException, ClassNotFoundException {
    QueryImpl q = new QueryImpl();
    q.setQueryLogicName("EventQuery");
    q.setExpirationDate(new Date());
    q.setId(UUID.randomUUID());
    q.setPagesize(10);
    q.setPageTimeout(-1);
    q.setQuery("FOO == BAR");
    q.setQueryName("test query");
    q.setQueryAuthorizations("ALL");
    q.setUserDN("some user");
    q.setOwner("some owner");
    q.setColumnVisibility("A&B");
    Set<Parameter> parameters = new HashSet<>();
    parameters.add(new Parameter("some param", "some value"));
    q.setParameters(parameters);
    Mutation m = QueryUtil.toMutation(q, new ColumnVisibility(q.getColumnVisibility()));
    Assert.assertEquals(1, m.getUpdates().size());
    byte[] value = m.getUpdates().get(0).getValue();
    Query q2 = QueryUtil.deserialize(QueryImpl.class.getName(), new Text("A&B"), new Value(value));
    Assert.assertEquals(q, q2);
}

Example 12 with Parameter

use of datawave.webservice.query.QueryImpl.Parameter in project datawave by NationalSecurityAgency.

the class QueryUtil method toParametersString.

public static String toParametersString(final Set<Parameter> parameters) {
    final StringBuilder params = new StringBuilder();
    if (null != parameters) {
        for (final Parameter param : parameters) {
            if (params.length() > 0) {
                params.append(PARAMETER_SEPARATOR);
            }
            params.append(param.getParameterName());
            params.append(PARAMETER_NAME_VALUE_SEPARATOR);
            params.append(param.getParameterValue());
        }
    }
    return params.toString();
}

Example 13 with Parameter

use of datawave.webservice.query.QueryImpl.Parameter in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method addDateFilters.

/**
 * Adding date filters if the query parameters specify that the dates are to be other than the default
 *
 * @param queryTree
 * @param scannerFactory
 * @param metadataHelper
 * @param config
 * @return the updated query tree
 * @throws TableNotFoundException
 */
public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, Query settings) throws TableNotFoundException, DatawaveQueryException {
    String defaultDateType = config.getDefaultDateTypeName();
    String dateType = defaultDateType;
    Parameter dateTypeParameter = settings.findParameter(QueryParameters.DATE_RANGE_TYPE);
    if (dateTypeParameter != null && dateTypeParameter.getParameterValue() != null) {
        String parm = dateTypeParameter.getParameterValue().trim();
        if (!parm.isEmpty()) {
            dateType = parm.toUpperCase();
        }
    }
    // time, then we need to modify the query
    if (!dateType.equals(defaultDateType)) {
        log.info("Using the date index for " + dateType);
        // if no date index helper configured, then we are in error
        if (dateIndexHelper == null) {
            throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
        }
        // get all of the fields used for this date type
        DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(), config.getDatatypeFilter());
        if (dateIndexData.getFields().isEmpty()) {
            log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
            // If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
            // Also it will be understandable from the plan as to why no results were returned.
            dateIndexData.getFields().add(Constants.NO_FIELD);
        }
        log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
        // now for each field, add an expression to filter that date
        List<JexlNode> andChildren = new ArrayList<>();
        for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
            andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
        }
        List<JexlNode> orChildren = new ArrayList<>();
        for (String field : dateIndexData.getFields()) {
            orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
        }
        if (orChildren.size() > 1) {
            andChildren.add(JexlNodeFactory.createOrNode(orChildren));
        } else {
            andChildren.addAll(orChildren);
        }
        JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
        JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));
        // now lets update the query parameters with the correct start and
        // end dates
        log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates " + dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());
        // reset the dates in the configuration, no need to reset then in
        // the Query settings object
        config.setBeginDate(dateIndexData.getBeginDate());
        config.setEndDate(dateIndexData.getEndDate());
    } else {
        log.info("Date index not needed for this query");
    }
    return queryTree;
}

Example 14 with Parameter

use of datawave.webservice.query.QueryImpl.Parameter in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method updateQueryTree.

protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataHelper metadataHelper, DateIndexHelper dateIndexHelper, ShardQueryConfiguration config, String query, QueryData queryData, Query settings) throws DatawaveQueryException {
    final QueryStopwatch timers = config.getTimers();
    TraceStopwatch stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - Parse query");
    config.setQueryTree(parseQueryAndValidatePattern(query, stopwatch));
    if (log.isDebugEnabled()) {
        logQuery(config.getQueryTree(), "Query after initial parse:");
    }
    stopwatch.stop();
    Map<String, String> optionsMap = new HashMap<>();
    if (query.contains(QueryFunctions.QUERY_FUNCTION_NAMESPACE + ':')) {
        // only do the extra tree visit if the function is present
        stopwatch = timers.newStartedStopwatch("DefaultQueryPlanner - parse out queryOptions from options function");
        config.setQueryTree(QueryOptionsFromQueryVisitor.collect(config.getQueryTree(), optionsMap));
        if (!optionsMap.isEmpty()) {
            QueryOptionsSwitch.apply(optionsMap, config);
        }
        stopwatch.stop();
    }
    // groom the query so that any nodes with the literal on the left and the identifier on
    // the right will be re-ordered to simplify subsequent processing
    config.setQueryTree(timedInvertSwappedNodes(timers, config.getQueryTree()));
    config.setQueryTree(timedFixNotNullIntent(timers, config.getQueryTree()));
    config.setQueryTree(timedIncludeDateFilters(timers, config.getQueryTree(), config, metadataHelper, scannerFactory, dateIndexHelper, settings));
    // note this must be called after we do the date adjustments per the query date type in addDateFilters
    timedCapDateRange(timers, config);
    // Find unmarked bounded ranges
    if (UnmarkedBoundedRangeDetectionVisitor.findUnmarkedBoundedRanges(config.getQueryTree())) {
        throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
    }
    if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
        config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
    } else {
        // look for the shards and days hint in the query settings
        // the shards and days hint cannot always be specified in the query string when using certain query parsers
        Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
        if (StringUtils.isNotBlank(parameter.getParameterValue())) {
            optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
            config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
        }
    }
    // extract #NO_EXPANSION function, if it exists
    config.setQueryTree(parseNoExpansionFields(timers, config.getQueryTree(), config));
    // flatten the tree
    config.setQueryTree(timedFlatten(timers, config.getQueryTree()));
    validateQuerySize("initial parse", config.getQueryTree(), config);
    config.setQueryTree(timedApplyRules(timers, config.getQueryTree(), config, metadataHelper, scannerFactory));
    config.setQueryTree(timedFixNegativeNumbers(timers, config.getQueryTree()));
    // Fix any query property markers that have multiple unwrapped sources.
    config.setQueryTree(timedFixQueryPropertyMarkers(timers, config.getQueryTree()));
    // Ensure that all ASTIdentifier nodes (field names) are upper-case to be consistent with what is enforced at ingest time
    config.setQueryTree(timedUpperCaseIdentifiers(timers, config.getQueryTree(), config, metadataHelper));
    config.setQueryTree(timedRewriteNegations(timers, config.getQueryTree()));
    QueryModel queryModel = loadQueryModel(config);
    config.setQueryTree(timedApplyQueryModel(timers, config.getQueryTree(), config, metadataHelper, queryModel));
    // Enforce unique terms within an AND or OR expression.
    if (config.getEnforceUniqueTermsWithinExpressions()) {
        config.setQueryTree(timedEnforceUniqueTermsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique AND'd terms within OR expressions.
    if (config.getEnforceUniqueConjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueConjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    // Enforce unique OR'd terms within AND expressions.
    if (config.getEnforceUniqueDisjunctionsWithinExpression()) {
        config.setQueryTree(timedEnforceUniqueDisjunctionsWithinExpressions(timers, config.getQueryTree()));
    }
    Set<String> indexOnlyFields = loadIndexedFields(config);
    if (disableBoundedLookup) {
        // LT,GT or ER node, we should expand it
        if (BoundedRangeDetectionVisitor.mustExpandBoundedRange(config, metadataHelper, config.getQueryTree()))
            disableBoundedLookup = false;
    }
    if (!indexOnlyFields.isEmpty()) {
        config.setQueryTree(expandRegexFunctionNodes(config.getQueryTree(), config, metadataHelper, indexOnlyFields));
    }
    config.setQueryTree(processTree(config.getQueryTree(), config, settings, metadataHelper, scannerFactory, queryData, timers, queryModel));
    // ExpandCompositeTerms was here
    boolean containsIndexOnlyFields = false;
    if (!indexOnlyFields.isEmpty() && !disableBoundedLookup) {
        // Figure out if the query contained any index only terms so we know
        // if we have to force it down the field-index path with event-specific
        // ranges
        containsIndexOnlyFields = timedCheckForIndexOnlyFieldsInQuery(timers, "Check for Index-Only Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper, indexOnlyFields);
    }
    boolean containsComposites = timedCheckForCompositeFields(timers, "Check for Composite Fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    boolean sortedUIDS = timedCheckForSortedUids(timers, "Check for Sorted UIDs", config.getQueryTree(), config);
    // check the query for any fields that are term frequencies
    // if any exist, populate the shard query config with these fields
    timedCheckForTokenizedFields(timers, "Check for term frequency (tokenized) fields", config.getQueryTree(), config, metadataHelper, dateIndexHelper);
    if (reduceQuery) {
        config.setQueryTree(timedReduce(timers, "Reduce Query Final", config.getQueryTree()));
    }
    return config.getQueryTree();
}

Example 15 with Parameter

use of datawave.webservice.query.QueryImpl.Parameter in project datawave by NationalSecurityAgency.

the class QueryExecutorBean method validateQuery.

/**
 * This method will provide some initial query validation for the define and create query calls.
 */
private QueryData validateQuery(String queryLogicName, MultivaluedMap<String, String> queryParameters, HttpHeaders httpHeaders) {
    // Parameter 'logicName' is required and passed in prior to this call. Add to the queryParameters now.
    if (!queryParameters.containsKey(QueryParameters.QUERY_LOGIC_NAME)) {
        queryParameters.putSingle(QueryParameters.QUERY_LOGIC_NAME, queryLogicName);
    }
    QueryData qd = new QueryData();
    log.debug(queryParameters);
    qp.clear();
    qp.setRequestHeaders(httpHeaders != null ? httpHeaders.getRequestHeaders() : null);
    // Pull "params" values into individual query parameters for validation on the query logic.
    // This supports the deprecated "params" value (both on the old and new API). Once we remove the deprecated
    // parameter, this code block can go away.
    String params = queryParameters.getFirst(QueryParameters.QUERY_PARAMS);
    if (params != null) {
        for (Parameter pm : QueryUtil.parseParameters(params)) {
            if (!queryParameters.containsKey(pm.getParameterName())) {
                queryParameters.putSingle(pm.getParameterName(), pm.getParameterValue());
            }
        }
    }
    queryParameters.remove(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ);
    queryParameters.remove(AuditParameters.USER_DN);
    queryParameters.remove(AuditParameters.QUERY_AUDIT_TYPE);
    // Ensure that all required parameters exist prior to validating the values.
    qp.validate(queryParameters);
    // Leaving for now until we can test to ensure that is always the case.
    if (qp.getPagesize() <= 0) {
        log.error("Invalid page size: " + qp.getPagesize());
        GenericResponse<String> response = new GenericResponse<>();
        throwBadRequest(DatawaveErrorCode.INVALID_PAGE_SIZE, response);
    }
    if (qp.getPageTimeout() != -1 && (qp.getPageTimeout() < PAGE_TIMEOUT_MIN || qp.getPageTimeout() > PAGE_TIMEOUT_MAX)) {
        log.error("Invalid page timeout: " + qp.getPageTimeout());
        GenericResponse<String> response = new GenericResponse<>();
        throwBadRequest(DatawaveErrorCode.INVALID_PAGE_TIMEOUT, response);
    }
    if (System.currentTimeMillis() >= qp.getExpirationDate().getTime()) {
        log.error("Invalid expiration date: " + qp.getExpirationDate());
        GenericResponse<String> response = new GenericResponse<>();
        throwBadRequest(DatawaveErrorCode.INVALID_EXPIRATION_DATE, response);
    }
    // Ensure begin date does not occur after the end date (if dates are not null)
    if ((qp.getBeginDate() != null && qp.getEndDate() != null) && qp.getBeginDate().after(qp.getEndDate())) {
        log.error("Invalid begin and/or end date: " + qp.getBeginDate() + " - " + qp.getEndDate());
        GenericResponse<String> response = new GenericResponse<>();
        throwBadRequest(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE, response);
    }
    // will throw IllegalArgumentException if not defined
    try {
        qd.logic = queryLogicFactory.getQueryLogic(queryLogicName, ctx.getCallerPrincipal());
    } catch (Exception e) {
        log.error("Failed to get query logic for " + queryLogicName, e);
        BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e);
        GenericResponse<String> response = new GenericResponse<>();
        response.addException(qe.getBottomQueryException());
        throw new BadRequestException(qe, response);
    }
    qd.logic.validate(queryParameters);
    try {
        marking.clear();
        marking.validate(queryParameters);
    } catch (IllegalArgumentException e) {
        log.error("Failed security markings validation", e);
        BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e);
        GenericResponse<String> response = new GenericResponse<>();
        response.addException(qe);
        throw new BadRequestException(qe, response);
    }
    // Find out who/what called this method
    qd.proxyServers = null;
    qd.p = ctx.getCallerPrincipal();
    qd.userDn = qd.p.getName();
    qd.userid = qd.userDn;
    qd.dnList = Collections.singletonList(qd.userid);
    if (qd.p instanceof DatawavePrincipal) {
        DatawavePrincipal dp = (DatawavePrincipal) qd.p;
        qd.userid = dp.getShortName();
        qd.userDn = dp.getUserDN().subjectDN();
        String[] dns = dp.getDNs();
        Arrays.sort(dns);
        qd.dnList = Arrays.asList(dns);
        qd.proxyServers = dp.getProxyServers();
        // Verify that the calling principal has access to the query logic.
        if (!qd.logic.containsDNWithAccess(qd.dnList)) {
            UnauthorizedQueryException qe = new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + qd.dnList, 401);
            GenericResponse<String> response = new GenericResponse<>();
            response.addException(qe);
            throw new UnauthorizedException(qe, response);
        }
    }
    log.trace(qd.userid + " has authorizations " + ((qd.p instanceof DatawavePrincipal) ? ((DatawavePrincipal) qd.p).getAuthorizations() : ""));
    // always check against the max
    if (qd.logic.getMaxPageSize() > 0 && qp.getPagesize() > qd.logic.getMaxPageSize()) {
        log.error("Invalid page size: " + qp.getPagesize() + " vs " + qd.logic.getMaxPageSize());
        BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", qd.logic.getMaxPageSize()));
        GenericResponse<String> response = new GenericResponse<>();
        response.addException(qe);
        throw new BadRequestException(qe, response);
    }
    // privileged users however can set whatever they want
    if (qp.isMaxResultsOverridden() && qd.logic.getMaxResults() >= 0) {
        if (!ctx.isCallerInRole(PRIVILEGED_USER)) {
            if (qp.getMaxResultsOverride() < 0 || (qd.logic.getMaxResults() < qp.getMaxResultsOverride())) {
                log.error("Invalid max results override: " + qp.getMaxResultsOverride() + " vs " + qd.logic.getMaxResults());
                GenericResponse<String> response = new GenericResponse<>();
                throwBadRequest(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE, response);
            }
        }
    }
    // Set private audit-related parameters, stripping off any that the user might have passed in first.
    // These are parameters that aren't passed in by the user, but rather are computed from other sources.
    PrivateAuditConstants.stripPrivateParameters(queryParameters);
    queryParameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName);
    queryParameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString());
    queryParameters.add(PrivateAuditConstants.USER_DN, qd.userDn);
    return qd;
}