Examples with ASTJexlScript org.apache.commons.jexl2.parser.ASTJexlScript used on opensource projects

Example 1 with ASTJexlScript

use of org.apache.commons.jexl2.parser.ASTJexlScript in project datawave by NationalSecurityAgency.

the class QueryJexl method createNormalizedExpression.

// =================================
// private methods
private Expression createNormalizedExpression(final String query) {
    try {
        ASTJexlScript script = JexlASTHelper.parseJexlQuery(query);
        Deque<SimpleNode> nodes = new LinkedList<>();
        normalizeScript(script, nodes);
        return new NormalizedExpression(jEngine, query, script);
    } catch (TokenMgrError | org.apache.commons.jexl2.parser.ParseException pe) {
        throw new AssertionError(pe);
    }
}

Example 2 with ASTJexlScript

use of org.apache.commons.jexl2.parser.ASTJexlScript in project datawave by NationalSecurityAgency.

the class QueryIterator method buildDocumentIterator.

/**
 * Build the document iterator
 *
 * @param documentRange
 * @param seekRange
 * @param columnFamilies
 * @param inclusive
 * @return
 * @throws IOException
 */
protected NestedIterator<Key> buildDocumentIterator(Range documentRange, Range seekRange, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException, ConfigException, InstantiationException, IllegalAccessException {
    NestedIterator<Key> docIter = null;
    if (log.isTraceEnabled()) {
        log.trace("Batched queries is " + batchedQueries);
    }
    if (batchedQueries >= 1) {
        List<NestedQuery<Key>> nests = Lists.newArrayList();
        for (Entry<Range, String> queries : batchStack) {
            Range myRange = queries.getKey();
            if (log.isTraceEnabled()) {
                log.trace("Adding " + myRange + " from seekrange " + seekRange);
            }
            /*
                 * Only perform the following checks if start key is not infinite and document range is specified
                 */
            if (null != seekRange && !seekRange.isInfiniteStartKey()) {
                Key seekStartKey = seekRange.getStartKey();
                Key myStartKey = myRange.getStartKey();
                /*
                     * if our seek key is greater than our start key we can skip this batched query. myStartKey.compareTo(seekStartKey) must be <= 0, which
                     * means that startKey must be greater than or equal be seekStartKey
                     */
                if (null != myStartKey && null != seekStartKey && !seekRange.contains(myStartKey)) {
                    if (log.isTraceEnabled()) {
                        log.trace("skipping " + myRange);
                    }
                    continue;
                }
            }
            JexlArithmetic myArithmetic;
            if (arithmetic instanceof StatefulArithmetic) {
                myArithmetic = ((StatefulArithmetic) arithmetic).clone();
            } else {
                myArithmetic = new DefaultArithmetic();
            }
            // Parse the query
            ASTJexlScript myScript = null;
            JexlEvaluation eval = null;
            try {
                myScript = JexlASTHelper.parseJexlQuery(queries.getValue());
                eval = new JexlEvaluation(queries.getValue(), myArithmetic);
            } catch (Exception e) {
                throw new IOException("Could not parse the JEXL query: '" + this.getQuery() + "'", e);
            }
            // If we had an event-specific range previously, we need to
            // reset it back
            // to the source we created during init
            NestedIterator<Key> subDocIter = getOrSetKeySource(myRange, myScript);
            if (log.isTraceEnabled()) {
                log.trace("Using init()'ialized source: " + subDocIter.getClass().getName());
            }
            if (gatherTimingDetails()) {
                subDocIter = new EvaluationTrackingNestedIterator(QuerySpan.Stage.FieldIndexTree, trackingSpan, subDocIter, myEnvironment);
            }
            // Seek() the boolean logic stuff
            ((SeekableIterator) subDocIter).seek(myRange, columnFamilies, inclusive);
            NestedQuery<Key> nestedQueryObj = new NestedQuery<>();
            nestedQueryObj.setQuery(queries.getValue());
            nestedQueryObj.setIterator(subDocIter);
            nestedQueryObj.setQueryScript(myScript);
            nestedQueryObj.setEvaluation(eval);
            nestedQueryObj.setRange(queries.getKey());
            nests.add(nestedQueryObj);
        }
        docIter = new NestedQueryIterator<>(nests);
        // now lets start off the nested iterator
        docIter.initialize();
        initKeySource = docIter;
    } else {
        // If we had an event-specific range previously, we need to reset it back
        // to the source we created during init
        docIter = getOrSetKeySource(documentRange, script);
        initKeySource = docIter;
        if (log.isTraceEnabled()) {
            log.trace("Using init()'ialized source: " + this.initKeySource.getClass().getName());
        }
        if (gatherTimingDetails()) {
            docIter = new EvaluationTrackingNestedIterator(QuerySpan.Stage.FieldIndexTree, trackingSpan, docIter, myEnvironment);
        }
        // Seek() the boolean logic stuff
        ((SeekableIterator) docIter).seek(range, columnFamilies, inclusive);
        // now lets start off the nested iterator
        docIter.initialize();
    }
    return docIter;
}

Example 3 with ASTJexlScript

use of org.apache.commons.jexl2.parser.ASTJexlScript in project datawave by NationalSecurityAgency.

the class EdgeQueryLogic method applyQueryModel.

/**
 * Parses the Jexl Query string into an ASTJexlScript and then uses QueryModelVisitor to apply queryModel to the query string, and then rewrites the
 * translated ASTJexlScript back to a query string using JexlStringBuildingVisitor.
 *
 * @param queryString
 * @return
 */
protected String applyQueryModel(String queryString) {
    ASTJexlScript origScript = null;
    ASTJexlScript script = null;
    try {
        origScript = JexlASTHelper.parseAndFlattenJexlQuery(queryString);
        HashSet<String> allFields = new HashSet<>();
        allFields.addAll(getEdgeQueryModel().getAllInternalFieldNames());
        script = QueryModelVisitor.applyModel(origScript, getEdgeQueryModel(), allFields);
        return JexlStringBuildingVisitor.buildQuery(script);
    } catch (Throwable t) {
        throw new IllegalStateException("Edge query model could not be applied", t);
    }
}

Example 4 with ASTJexlScript

use of org.apache.commons.jexl2.parser.ASTJexlScript in project datawave by NationalSecurityAgency.

the class VisitorFunction method apply.

@Override
@Nullable
public ScannerChunk apply(@Nullable ScannerChunk input) {
    SessionOptions options = input.getOptions();
    ScannerChunk newSettings = new ScannerChunk(null, input.getRanges(), input.getLastKnownLocation());
    SessionOptions newOptions = new SessionOptions(options);
    for (IteratorSetting setting : options.getIterators()) {
        final String query = setting.getOptions().get(QueryOptions.QUERY);
        if (null != query) {
            IteratorSetting newIteratorSetting = new IteratorSetting(setting.getPriority(), setting.getName(), setting.getIteratorClass());
            newIteratorSetting.addOptions(setting.getOptions());
            try {
                ASTJexlScript script = null;
                boolean evaluatedPreviously = previouslyExecutable(query);
                boolean madeChange = false;
                if (!evaluatedPreviously && config.isCleanupShardsAndDaysQueryHints()) {
                    script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    script = DateIndexCleanupVisitor.cleanup(script);
                    madeChange = true;
                }
                String newQuery = evaluatedPreviously ? previouslyExpanded.get(query) : query;
                List<String> debug = null;
                if (log.isTraceEnabled())
                    debug = Lists.newArrayList();
                if (!config.isDisableWhindexFieldMappings() && !evaluatedPreviously) {
                    if (null == script)
                        script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    // apply the whindex using the shard date
                    ASTJexlScript rebuiltScript = WhindexVisitor.apply(script, config, getEarliestBeginDate(newSettings.getRanges()), metadataHelper);
                    // if the query changed, save it, and mark it as such
                    if (!TreeEqualityVisitor.isEqual(script, rebuiltScript)) {
                        log.debug("[" + config.getQuery().getId() + "] The WhindexVisitor updated the query: " + JexlStringBuildingVisitor.buildQuery(script));
                        script = rebuiltScript;
                        madeChange = true;
                    }
                }
                if (!config.isBypassExecutabilityCheck() || !evaluatedPreviously) {
                    if (null == script)
                        script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                    if (!ExecutableDeterminationVisitor.isExecutable(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, this.metadataHelper)) {
                        if (log.isTraceEnabled()) {
                            log.trace("Need to pull up non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
                            for (String debugStatement : debug) {
                                log.trace(debugStatement);
                            }
                            DefaultQueryPlanner.logQuery(script, "Failing query:");
                        }
                        script = (ASTJexlScript) PullupUnexecutableNodesVisitor.pullupDelayedPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
                        madeChange = true;
                        STATE state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
                        /**
                         * We could achieve better performance if we live with the small number of queries that error due to the full table scan exception.
                         *
                         * Either look at improving PushdownUnexecutableNodesVisitor or avoid the process altogether.
                         */
                        if (state != STATE.EXECUTABLE) {
                            if (log.isTraceEnabled()) {
                                log.trace("Need to push down non-executable query: " + JexlStringBuildingVisitor.buildQuery(script));
                                for (String debugStatement : debug) {
                                    log.trace(debugStatement);
                                }
                            }
                            script = (ASTJexlScript) PushdownUnexecutableNodesVisitor.pushdownPredicates(script, true, config, indexedFields, indexOnlyFields, nonEventFields, metadataHelper);
                        }
                        state = ExecutableDeterminationVisitor.getState(script, config, indexedFields, indexOnlyFields, nonEventFields, true, debug, metadataHelper);
                        if (state != STATE.EXECUTABLE) {
                            if (state == STATE.ERROR) {
                                log.warn("After expanding the query, it is determined that the query cannot be executed due to index-only fields mixed with expressions that cannot be run against the index.");
                                BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INDEX_ONLY_FIELDS_MIXED_INVALID_EXPRESSIONS);
                                throw new InvalidQueryException(qe);
                            }
                            log.warn("After expanding the query, it is determined that the query cannot be executed against the field index and a full table scan is required");
                            if (!config.getFullTableScanEnabled()) {
                                if (log.isTraceEnabled()) {
                                    log.trace("Full Table fail of " + JexlStringBuildingVisitor.buildQuery(script));
                                    for (String debugStatement : debug) {
                                        log.trace(debugStatement);
                                    }
                                    DefaultQueryPlanner.logQuery(script, "Failing query:");
                                }
                                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.FULL_TABLE_SCAN_REQUIRED_BUT_DISABLED);
                                throw new DatawaveFatalQueryException(qe);
                            }
                        }
                        if (log.isTraceEnabled()) {
                            for (String debugStatement : debug) {
                                log.trace(debugStatement);
                            }
                            DefaultQueryPlanner.logQuery(script, "Query pushing down large fielded lists:");
                        }
                    }
                }
                if (config.getSerializeQueryIterator()) {
                    serializeQuery(newIteratorSetting);
                } else {
                    if (!evaluatedPreviously) {
                        // if we have an hdfs configuration, then we can pushdown large fielded lists to an ivarator
                        if (config.getHdfsSiteConfigURLs() != null && setting.getOptions().get(QueryOptions.BATCHED_QUERY) == null) {
                            if (null == script)
                                script = JexlASTHelper.parseAndFlattenJexlQuery(query);
                            try {
                                script = pushdownLargeFieldedLists(config, script);
                                madeChange = true;
                            } catch (IOException ioe) {
                                log.error("Unable to pushdown large fielded lists....leaving in expanded form", ioe);
                            }
                        }
                    }
                }
                // only recompile the script if changes were made to the query
                if (madeChange)
                    newQuery = JexlStringBuildingVisitor.buildQuery(script);
                try {
                    previouslyExpanded.put(query, newQuery);
                } catch (NullPointerException npe) {
                    throw new DatawaveFatalQueryException(String.format("New query is null! madeChange: %b, qid: %s", madeChange, setting.getOptions().get(QueryOptions.QUERY_ID)), npe);
                }
                // test the final script for thresholds
                DefaultQueryPlanner.validateQuerySize("VisitorFunction", script, config, false);
                newIteratorSetting.addOption(QueryOptions.QUERY, newQuery);
                newOptions.removeScanIterator(setting.getName());
                newOptions.addScanIterator(newIteratorSetting);
                if (log.isDebugEnabled()) {
                    log.debug("VisitorFunction result: " + newSettings.getRanges());
                }
                if (log.isTraceEnabled()) {
                    DefaultQueryPlanner.logTrace(PrintingVisitor.formattedQueryStringList(script), "VistorFunction::apply method");
                } else if (log.isDebugEnabled()) {
                    DefaultQueryPlanner.logDebug(PrintingVisitor.formattedQueryStringList(script, DefaultQueryPlanner.maxChildNodesToPrint), "VistorFunction::apply method");
                }
            } catch (ParseException e) {
                throw new DatawaveFatalQueryException(e);
            }
        }
    }
    newSettings.setOptions(newOptions);
    return newSettings;
}

Example 5 with ASTJexlScript

use of org.apache.commons.jexl2.parser.ASTJexlScript in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTable method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    ShardIndexQueryConfiguration config = new ShardIndexQueryConfiguration(this, settings);
    this.scannerFactory = new ScannerFactory(connection);
    MetadataHelper metadataHelper = initializeMetadataHelper(connection, config.getMetadataTableName(), auths);
    if (StringUtils.isEmpty(settings.getQuery())) {
        throw new IllegalArgumentException("Query cannot be null");
    }
    if (log.isDebugEnabled()) {
        log.debug("Query parameters set to " + settings.getParameters());
    }
    String tModelName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_NAME);
    if (tModelName != null) {
        modelName = tModelName;
    }
    String tModelTableName = getTrimmedOrNull(settings, QueryParameters.PARAMETER_MODEL_TABLE_NAME);
    if (tModelTableName != null) {
        modelTableName = tModelTableName;
    }
    queryModel = metadataHelper.getQueryModel(modelTableName, modelName, null);
    String datatypeFilterString = getTrimmedOrNull(settings, QueryParameters.DATATYPE_FILTER_SET);
    if (datatypeFilterString != null) {
        config.setDatatypeFilter(new HashSet<>(Arrays.asList(datatypeFilterString.split(PARAM_VALUE_SEP_STR))));
        if (log.isDebugEnabled()) {
            log.debug("Data type filter set to " + config.getDatatypeFilterAsString());
        }
    }
    config.setConnector(connection);
    config.setAuthorizations(auths);
    if (indexTableName != null) {
        config.setIndexTableName(indexTableName);
    }
    if (reverseIndexTableName != null) {
        config.setReverseIndexTableName(reverseIndexTableName);
    }
    if (settings.getBeginDate() != null) {
        config.setBeginDate(settings.getBeginDate());
    } else {
        config.setBeginDate(new Date(0));
        if (log.isDebugEnabled()) {
            log.debug("No begin date supplied in settings.");
        }
    }
    if (settings.getEndDate() != null) {
        config.setEndDate(settings.getEndDate());
    } else {
        config.setEndDate(new Date(Long.MAX_VALUE));
        if (log.isDebugEnabled()) {
            log.debug("No end date supplied in settings.");
        }
    }
    // start with a trimmed version of the query, converted to JEXL
    LuceneToJexlQueryParser parser = new LuceneToJexlQueryParser();
    parser.setAllowLeadingWildCard(this.isAllowLeadingWildcard());
    QueryNode node = parser.parse(settings.getQuery().trim());
    // TODO: Validate that this is a simple list of terms type of query
    config.setQueryString(node.getOriginalQuery());
    if (log.isDebugEnabled()) {
        log.debug("Original Query = " + settings.getQuery().trim());
        log.debug("JEXL Query = " + node.getOriginalQuery());
    }
    // Parse & flatten the query.
    ASTJexlScript origScript = JexlASTHelper.parseAndFlattenJexlQuery(config.getQueryString());
    ASTJexlScript script;
    try {
        script = UnfieldedIndexExpansionVisitor.expandUnfielded(config, this.scannerFactory, metadataHelper, origScript);
    } catch (EmptyUnfieldedTermExpansionException e) {
        Multimap<String, String> emptyMap = Multimaps.unmodifiableMultimap(HashMultimap.create());
        config.setNormalizedTerms(emptyMap);
        config.setNormalizedPatterns(emptyMap);
        return config;
    }
    Set<String> dataTypes = config.getDatatypeFilter();
    Set<String> allFields = metadataHelper.getAllFields(dataTypes);
    script = QueryModelVisitor.applyModel(script, queryModel, allFields);
    if (log.isTraceEnabled()) {
        log.trace("fetching dataTypes from FetchDataTypesVisitor");
    }
    Multimap<String, Type<?>> fieldToDataTypeMap = FetchDataTypesVisitor.fetchDataTypes(metadataHelper, config.getDatatypeFilter(), script);
    config.setDataTypes(fieldToDataTypeMap);
    config.setQueryFieldsDatatypes(fieldToDataTypeMap);
    final Set<String> indexedFields = metadataHelper.getIndexedFields(dataTypes);
    config.setIndexedFields(indexedFields);
    final Set<String> reverseIndexedFields = metadataHelper.getReverseIndexedFields(dataTypes);
    config.setReverseIndexedFields(reverseIndexedFields);
    final Multimap<String, Type<?>> normalizedFields = metadataHelper.getFieldsToDatatypes(dataTypes);
    config.setNormalizedFieldsDatatypes(normalizedFields);
    if (log.isTraceEnabled()) {
        log.trace("Normalizers:");
        for (String field : fieldToDataTypeMap.keySet()) {
            log.trace(field + ": " + fieldToDataTypeMap.get(field));
        }
    }
    script = ExpandMultiNormalizedTerms.expandTerms(config, metadataHelper, script);
    Multimap<String, String> literals = LiteralNodeVisitor.getLiterals(script);
    Multimap<String, String> patterns = PatternNodeVisitor.getPatterns(script);
    Map<Entry<String, String>, Range> rangesForTerms = Maps.newHashMap();
    Map<Entry<String, String>, Entry<Range, Boolean>> rangesForPatterns = Maps.newHashMap();
    config.setNormalizedTerms(literals);
    config.setNormalizedPatterns(patterns);
    if (log.isDebugEnabled()) {
        log.debug("Normalized Literals = " + literals);
        log.debug("Normalized Patterns = " + patterns);
    }
    for (Entry<String, String> entry : literals.entries()) {
        rangesForTerms.put(entry, ShardIndexQueryTableStaticMethods.getLiteralRange(entry));
    }
    for (Entry<String, String> entry : patterns.entries()) {
        ShardIndexQueryTableStaticMethods.RefactoredRangeDescription r = ShardIndexQueryTableStaticMethods.getRegexRange(entry, isFullTableScanEnabled(), metadataHelper, config);
        rangesForPatterns.put(entry, Maps.immutableEntry(r.range, r.isForReverseIndex));
    }
    config.setRangesForTerms(rangesForTerms);
    config.setRangesForPatterns(rangesForPatterns);
    return config;
}