Examples with QueryNode datawave.query.language.tree.QueryNode used on opensource projects

Example 1 with QueryNode

use of datawave.query.language.tree.QueryNode in project datawave by NationalSecurityAgency.

the class ShardQueryLogic method getJexlQueryString.

public String getJexlQueryString(Query settings) throws ParseException {
    // queryString should be JEXl after all query parsers are applied
    String queryString;
    String originalQuery = settings.getQuery();
    originalQuery = this.expandQueryMacros(originalQuery);
    // Determine query syntax (i.e. JEXL, LUCENE, etc.)
    String querySyntax = settings.findParameter(QueryParameters.QUERY_SYNTAX).getParameterValue();
    // enforce mandatoryQuerySyntax if set
    if (null != this.mandatoryQuerySyntax) {
        if (org.apache.commons.lang.StringUtils.isEmpty(querySyntax)) {
            throw new IllegalStateException("Must specify one of the following syntax options: " + this.mandatoryQuerySyntax);
        } else {
            if (!this.mandatoryQuerySyntax.contains(querySyntax)) {
                throw new IllegalStateException("Syntax not supported, must be one of the following: " + this.mandatoryQuerySyntax + ", submitted: " + querySyntax);
            }
        }
    }
    QueryParser querySyntaxParser = getParser();
    if (org.apache.commons.lang.StringUtils.isBlank(querySyntax)) {
        // Falling back to Jexl when one is not set on this class
        if (null == querySyntaxParser) {
            querySyntax = "JEXL";
        }
    } else if (!"JEXL".equals(querySyntax)) {
        if (null == querySyntaxParsers) {
            throw new IllegalStateException("Query syntax parsers not configured");
        }
        querySyntaxParser = querySyntaxParsers.get(querySyntax);
        if (null == querySyntaxParser) {
            // No parser was specified, try to default to the parser on the
            // class
            querySyntaxParser = getParser();
            if (null == querySyntaxParser) {
                throw new IllegalArgumentException("QueryParser not configured for syntax: " + querySyntax);
            }
        }
    }
    if (null == originalQuery) {
        throw new IllegalArgumentException("Query cannot be null");
    } else {
        if ("JEXL".equals(querySyntax)) {
            queryString = originalQuery;
        } else {
            QueryNode node = querySyntaxParser.parse(originalQuery);
            queryString = node.getOriginalQuery();
            if (log.isTraceEnabled()) {
                log.trace("luceneQueryString: " + originalQuery + " --> jexlQueryString: " + queryString);
            }
        }
    }
    return queryString;
}

Example 2 with QueryNode

use of datawave.query.language.tree.QueryNode in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTable method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    ShardIndexQueryConfiguration config = new ShardIndexQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = initializeMetadataHelper(connection, config.getMetadataTableName(), auths);
    if (StringUtils.isEmpty(settings.getQuery())) {
        throw new IllegalArgumentException("Query cannot be null");
    }
    if (log.isDebugEnabled()) {
        log.debug("Query parameters set to " + settings.getParameters());
    }
    String tModelName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_NAME);
    if (tModelName != null) {
        modelName = tModelName;
    }
    String tModelTableName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_TABLE_NAME);
    if (tModelTableName != null) {
        modelTableName = tModelTableName;
    }
    queryModel = metadataHelper.getQueryModel(modelTableName, modelName, null);
    String datatypeFilterString = getTrimmedOrNull(settings, QueryParameters.DATATYPE_FILTER_SET);
    if (datatypeFilterString != null) {
        config.setDatatypeFilter(new HashSet<>(Arrays.asList(datatypeFilterString.split(PARAM_VALUE_SEP_STR))));
        if (log.isDebugEnabled()) {
            log.debug("Data type filter set to " + config.getDatatypeFilterAsString());
        }
    }
    config.setConnector(connection);
    config.setAuthorizations(auths);
    if (indexTableName != null) {
        config.setIndexTableName(indexTableName);
    }
    if (reverseIndexTableName != null) {
        config.setReverseIndexTableName(reverseIndexTableName);
    }
    if (settings.getBeginDate() != null) {
        config.setBeginDate(settings.getBeginDate());
    } else {
        config.setBeginDate(new Date(0));
        if (log.isDebugEnabled()) {
            log.debug("No begin date supplied in settings.");
        }
    }
    if (settings.getEndDate() != null) {
        config.setEndDate(settings.getEndDate());
    } else {
        config.setEndDate(new Date(Long.MAX_VALUE));
        if (log.isDebugEnabled()) {
            log.debug("No end date supplied in settings.");
        }
    }
    // start with a trimmed version of the query, converted to JEXL
    LuceneToJexlQueryParser parser = new LuceneToJexlQueryParser();
    parser.setAllowLeadingWildCard(this.isAllowLeadingWildcard());
    QueryNode node = parser.parse(settings.getQuery().trim());
    // TODO: Validate that this is a simple list of terms type of query
    config.setQueryString(node.getOriginalQuery());
    if (log.isDebugEnabled()) {
        log.debug("Original Query = " + settings.getQuery().trim());
        log.debug("JEXL Query = " + node.getOriginalQuery());
    }
    // Parse & flatten the query.
    ASTJexlScript origScript = JexlASTHelper.parseAndFlattenJexlQuery(config.getQueryString());
    ASTJexlScript script;
    try {
        script = UnfieldedIndexExpansionVisitor.expandUnfielded(config, this.scannerFactory, metadataHelper, origScript);
    } catch (EmptyUnfieldedTermExpansionException e) {
        Multimap<String, String> emptyMap = Multimaps.unmodifiableMultimap(HashMultimap.create());
        config.setNormalizedTerms(emptyMap);
        config.setNormalizedPatterns(emptyMap);
        return config;
    }
    Set<String> dataTypes = config.getDatatypeFilter();
    Set<String> allFields = metadataHelper.getAllFields(dataTypes);
    script = QueryModelVisitor.applyModel(script, queryModel, allFields);
    if (log.isTraceEnabled()) {
        log.trace("fetching dataTypes from FetchDataTypesVisitor");
    }
    Multimap<String, Type<?>> fieldToDataTypeMap = FetchDataTypesVisitor.fetchDataTypes(metadataHelper, config.getDatatypeFilter(), script);
    config.setDataTypes(fieldToDataTypeMap);
    config.setQueryFieldsDatatypes(fieldToDataTypeMap);
    final Set<String> indexedFields = metadataHelper.getIndexedFields(dataTypes);
    config.setIndexedFields(indexedFields);
    final Set<String> reverseIndexedFields = metadataHelper.getReverseIndexedFields(dataTypes);
    config.setReverseIndexedFields(reverseIndexedFields);
    final Multimap<String, Type<?>> normalizedFields = metadataHelper.getFieldsToDatatypes(dataTypes);
    config.setNormalizedFieldsDatatypes(normalizedFields);
    if (log.isTraceEnabled()) {
        log.trace("Normalizers:");
        for (String field : fieldToDataTypeMap.keySet()) {
            log.trace(field + ": " + fieldToDataTypeMap.get(field));
        }
    }
    script = ExpandMultiNormalizedTerms.expandTerms(config, metadataHelper, script);
    Multimap<String, String> literals = LiteralNodeVisitor.getLiterals(script);
    Multimap<String, String> patterns = PatternNodeVisitor.getPatterns(script);
    Map<Entry<String, String>, Range> rangesForTerms = Maps.newHashMap();
    Map<Entry<String, String>, Entry<Range, Boolean>> rangesForPatterns = Maps.newHashMap();
    config.setNormalizedTerms(literals);
    config.setNormalizedPatterns(patterns);
    if (log.isDebugEnabled()) {
        log.debug("Normalized Literals = " + literals);
        log.debug("Normalized Patterns = " + patterns);
    }
    for (Entry<String, String> entry : literals.entries()) {
        rangesForTerms.put(entry, ShardIndexQueryTableStaticMethods.getLiteralRange(entry));
    }
    for (Entry<String, String> entry : patterns.entries()) {
        ShardIndexQueryTableStaticMethods.RefactoredRangeDescription r = ShardIndexQueryTableStaticMethods.getRegexRange(entry, isFullTableScanEnabled(), metadataHelper, config);
        rangesForPatterns.put(entry, Maps.immutableEntry(r.range, r.isForReverseIndex));
    }
    config.setRangesForTerms(rangesForTerms);
    config.setRangesForPatterns(rangesForPatterns);
    return config;
}

Example 3 with QueryNode

use of datawave.query.language.tree.QueryNode in project datawave by NationalSecurityAgency.

the class LuceneToJexlUUIDQueryParser method parse.

@Override
public QueryNode parse(String query) throws ParseException {
    // replace open smart quote 147
    query = query.replaceAll("\\u0093", "\"");
    // replace close smart quote 148
    query = query.replaceAll("\\u0094", "\"");
    QueryNode parsedQuery = null;
    parsedQuery = luceneParser.parse(query);
    if (!validUUIDQuery(parsedQuery))
        throw new ParseException("Query: " + query + " not supported with the LuceneToJexlUUIDQueryParser");
    return super.parse(query);
}

Example 4 with QueryNode

use of datawave.query.language.tree.QueryNode in project datawave by NationalSecurityAgency.

the class JexlControlledQueryParser method parse.

@Override
public QueryNode parse(String query) throws ParseException {
    checkIfQueryAllowed(query);
    StringBuilder sb = new StringBuilder();
    if (!includedValues.isEmpty()) {
        sb.append("(");
    }
    boolean addedFirstInclude = false;
    for (Map.Entry<String, Set<String>> entry : includedValues.entrySet()) {
        String field = entry.getKey();
        for (String value : entry.getValue()) {
            if (addedFirstInclude) {
                sb.append(" || ");
            }
            addedFirstInclude = true;
            sb.append("filter:includeRegex(").append(field).append(", '").append(value).append("')");
        }
    }
    if (!includedValues.isEmpty()) {
        sb.append(")");
    }
    if (!excludedValues.isEmpty()) {
        if (!includedValues.isEmpty()) {
            sb.append(" && ");
        }
        sb.append("(");
    }
    boolean addedFirstExclude = false;
    for (Map.Entry<String, Set<String>> entry : excludedValues.entrySet()) {
        String field = entry.getKey();
        for (String value : entry.getValue()) {
            if (addedFirstExclude) {
                sb.append(" && ");
            }
            addedFirstExclude = true;
            sb.append("not(filter:includeRegex(").append(field).append(", '").append(value).append("'))");
        }
    }
    if (!excludedValues.isEmpty()) {
        sb.append(")");
    }
    if (sb.length() > 0) {
        query = "(" + query + ")" + " && (" + sb + ")";
    }
    QueryNode node = new ServerHeadNode();
    node.setOriginalQuery(query);
    return node;
}

Example 5 with QueryNode

use of datawave.query.language.tree.QueryNode in project datawave by NationalSecurityAgency.

the class TestLuceneToJexlParser method test1.

@Test
public void test1() throws Exception {
    LuceneToJexlQueryParser parser = getQueryParser();
    QueryNode node = parser.parse("FIELD:SELECTOR AND #EXCLUDE(AND, F1, GB.*, F2, GB.*)");
    Assert.assertEquals("FIELD == 'SELECTOR' && (not(filter:includeRegex(F1, 'GB.*')) || not(filter:includeRegex(F2, 'GB.*')))", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #INCLUDE(AND, F1, GB.*, F2, GB.*)");
    Assert.assertEquals("FIELD == 'SELECTOR' && (filter:includeRegex(F1, 'GB.*') && filter:includeRegex(F2, 'GB.*'))", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #loaded(after, 20140101)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:afterLoadDate(LOAD_DATE, '20140101')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #loaded(before, 20140101, yyyyMMdd)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:beforeLoadDate(LOAD_DATE, '20140101', 'yyyyMMdd')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #loaded(between, 20140101, 20140102, yyyyMMdd)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:betweenLoadDates(LOAD_DATE, '20140101', '20140102', 'yyyyMMdd')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #date(SOME_DATE, after, 20140101)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:afterDate(SOME_DATE, '20140101')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #date(SOME_DATE, before, 20140101, yyyyMMdd)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:beforeDate(SOME_DATE, '20140101', 'yyyyMMdd')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #date(SOME_DATE, between, 20140101, 20140102, yyyyMMdd)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:betweenDates(SOME_DATE, '20140101', '20140102', 'yyyyMMdd')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #MATCHES_IN_GROUP(FIELD1, v1, FIELD2, v2)");
    Assert.assertEquals("FIELD == 'SELECTOR' && grouping:matchesInGroup(FIELD1, 'v1', FIELD2, 'v2')", node.getOriginalQuery());
    node = parser.parse("FIELD:SELECTOR AND #MATCHES_AT_LEAST_COUNT_OF(7, FIELD, v1, v2)");
    Assert.assertEquals("FIELD == 'SELECTOR' && filter:matchesAtLeastCountOf(7, FIELD, 'v1', 'v2')", node.getOriginalQuery());
    node = parser.parse("FIELD:SOMETHING AND #EVALUATION_ONLY('NUM_FIELD: [0 TO 50}')");
    Assert.assertEquals("FIELD == 'SOMETHING' && ((_Eval_ = true) && (((_Bounded_ = true) && (NUM_FIELD >= '0' && NUM_FIELD < '50'))))", node.getOriginalQuery());
}