Search in sources :

Example 1 with ASTAndNode

use of org.apache.commons.jexl2.parser.ASTAndNode in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitor method canExpand.

/**
 * Test if the current node should be expanded by this visitor. Update the tracker with the current andChild if expansion is possible
 *
 * @param node
 *            the node to test
 * @param tracker
 *            the tracker holding supplementary information about the expansion
 * @return true if the expansion should occur, false otherwise
 */
private boolean canExpand(JexlNode node, ExecutableDeterminationVisitor.STATE state, ExpansionTracker tracker) {
    // only process if state is ERROR or PARTIAL
    if (!(state == ExecutableDeterminationVisitor.STATE.ERROR || state == ExecutableDeterminationVisitor.STATE.PARTIAL)) {
        return false;
    }
    // if deeper down in the tree there was a failed expansion attempt, don't try again
    if (tracker.isFailedExpansion()) {
        return false;
    }
    // there must be an andNode further up the tree to distribute into the or
    ASTAndNode lastAnd = tracker.getLastAnd();
    // as long as there is a lastAnd there is work to do
    if (lastAnd == null) {
        return false;
    }
    // verify there is nothing but compatible nodes between the andNode and the orNode
    JexlNode current = node.jjtGetParent();
    JexlNode last = node;
    while (current != lastAnd) {
        if (!(current instanceof ASTReference || current instanceof ASTReferenceExpression)) {
            return false;
        }
        last = current;
        current = current.jjtGetParent();
    }
    // if we got here the expansion is good and we should track the andChild for later use
    tracker.setAndChild(last);
    return true;
}
Also used : ASTReferenceExpression(org.apache.commons.jexl2.parser.ASTReferenceExpression) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ASTReference(org.apache.commons.jexl2.parser.ASTReference) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode)

Example 2 with ASTAndNode

use of org.apache.commons.jexl2.parser.ASTAndNode in project datawave by NationalSecurityAgency.

the class TreeFlatteningRebuilder method copyTree.

/**
 * Iteratively creates a copy of the passed in JexlNode
 *
 * @param node
 *            the node to be copied
 * @param postOrderDeque
 *            the post order traversal of the copied tree
 * @return the copied tree
 */
private JexlNode copyTree(JexlNode node, Deque<JexlNode> postOrderDeque) {
    // add all the nodes to the stack and iterate...
    Deque<JexlNode> workingStack = new LinkedList<>();
    // create a copy of this node which shares the same children as the original node
    JexlNode copiedNode = rebuildNode(node);
    workingStack.push(copiedNode);
    // compute the post order traversal of all of the nodes, and copy them
    while (!workingStack.isEmpty()) {
        JexlNode poppedNode = workingStack.pop();
        postOrderDeque.push(poppedNode);
        // if this node has children, create copies of them
        if (poppedNode.jjtGetNumChildren() > 0) {
            List<JexlNode> copiedChildren = new ArrayList<>();
            List<JexlNode> children = (poppedNode instanceof ASTAndNode || poppedNode instanceof ASTOrNode) ? getAndOrLeaves(poppedNode) : Arrays.asList(children(poppedNode));
            for (JexlNode child : children) {
                if (child != null) {
                    // create a copy of this node which shares the same children as the original node
                    JexlNode copiedChild = rebuildNode(child);
                    copiedChildren.add(copiedChild);
                    workingStack.push(copiedChild);
                }
            }
            // Reassign the children for this copied node
            JexlNodes.children(poppedNode, copiedChildren.toArray(new JexlNode[0]));
        }
    }
    return copiedNode;
}
Also used : ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) ArrayList(java.util.ArrayList) JexlNode(org.apache.commons.jexl2.parser.JexlNode) LinkedList(java.util.LinkedList) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode)

Example 3 with ASTAndNode

use of org.apache.commons.jexl2.parser.ASTAndNode in project datawave by NationalSecurityAgency.

the class TreeFlatteningRebuilder method getAndOrLeaves.

private List<JexlNode> getAndOrLeaves(JexlNode node) {
    LinkedList<JexlNode> children = new LinkedList<>();
    LinkedList<JexlNode> stack = new LinkedList<>();
    stack.push(node);
    while (!stack.isEmpty()) {
        JexlNode currNode = stack.pop();
        // @formatter:off
        if (currNode == node || (node.getClass().isInstance(currNode) && (currNode instanceof ASTOrNode || (currNode instanceof ASTAndNode && !isBoundedRange((ASTAndNode) currNode))))) {
            // @formatter:on
            for (JexlNode child : children(currNode)) {
                stack.push(child);
            }
        } else {
            children.push(currNode);
        }
    }
    return children;
}
Also used : ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) JexlNode(org.apache.commons.jexl2.parser.JexlNode) LinkedList(java.util.LinkedList) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode)

Example 4 with ASTAndNode

use of org.apache.commons.jexl2.parser.ASTAndNode in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitor method visit.

@Override
public Object visit(ASTAndNode and, Object data) {
    QueryPropertyMarker.Instance instance = QueryPropertyMarker.findInstance(and);
    if (instance.isType(ExceededOrThresholdMarkerJexlNode.class)) {
        JexlNode source = instance.getSource();
        // Ivarator to get the job done
        if (source instanceof ASTAndNode) {
            try {
                ivarateList(and, source, data);
            } catch (IOException ioe) {
                throw new DatawaveFatalQueryException(ioe);
            }
        } else {
            QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "Limited ExceededOrThresholdMarkerJexlNode"));
            throw new DatawaveFatalQueryException(qe);
        }
    } else if (data instanceof IndexRangeIteratorBuilder) {
        // index checking has already been done, otherwise we would not have
        // an "ExceededValueThresholdMarker"
        // hence the "IndexAgnostic" method can be used here
        LiteralRange range = JexlASTHelper.findRange().recursively().getRange(and);
        if (range == null) {
            QueryException qe = new QueryException(DatawaveErrorCode.MULTIPLE_RANGES_IN_EXPRESSION);
            throw new DatawaveFatalQueryException(qe);
        }
        ((IndexRangeIteratorBuilder) data).setRange(range);
    } else if (instance.isType(ExceededValueThresholdMarkerJexlNode.class)) {
        // if the parent is our ExceededValueThreshold marker, then use an
        // Ivarator to get the job done unless we don't have to
        JexlNode source = instance.getSource();
        String identifier = null;
        LiteralRange<?> range = null;
        boolean negatedLocal = false;
        if (source instanceof ASTAndNode) {
            range = buildLiteralRange(source, null);
            identifier = range.getFieldName();
        } else {
            if (source instanceof ASTNRNode || source instanceof ASTNotNode)
                negatedLocal = true;
            range = buildLiteralRange(source);
            identifier = JexlASTHelper.getIdentifier(source);
        }
        boolean negatedOverall = negatedLocal;
        if (data instanceof AbstractIteratorBuilder) {
            AbstractIteratorBuilder oib = (AbstractIteratorBuilder) data;
            if (oib.isInANot()) {
                negatedOverall = !negatedOverall;
            }
        }
        // or the field is index only but not in the term frequencies, then we must ivarate
        if (!limitLookup || !allowTermFrequencyLookup || (indexOnlyFields.contains(identifier) && !termFrequencyFields.contains(identifier))) {
            if (source instanceof ASTAndNode) {
                try {
                    List<ASTFunctionNode> functionNodes = JexlASTHelper.getFunctionNodes(source).stream().filter(node -> JexlFunctionArgumentDescriptorFactory.F.getArgumentDescriptor(node).allowIvaratorFiltering()).collect(Collectors.toList());
                    if (functionNodes.isEmpty()) {
                        ivarateRange(and, source, data);
                    } else {
                        ivarateFilter(and, source, data, functionNodes);
                    }
                } catch (IOException ioe) {
                    throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
                }
            } else if (source instanceof ASTERNode || source instanceof ASTNRNode) {
                try {
                    ivarateRegex(and, source, data);
                } catch (IOException ioe) {
                    throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
                }
            } else {
                QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "ExceededValueThresholdMarkerJexlNode"));
                throw new DatawaveFatalQueryException(qe);
            }
        } else {
            NestedIterator<Key> nested = null;
            if (termFrequencyFields.contains(identifier)) {
                nested = buildExceededFromTermFrequency(identifier, and, source, range, data);
            } else {
                /**
                 * This is okay since 1) We are doc specific 2) We are not index only or tf 3) Therefore, we must evaluate against the document for this
                 * expression 4) Return a stubbed range in case we have a disjunction that breaks the current doc.
                 */
                if (!limitOverride && !negatedOverall)
                    nested = createExceededCheck(identifier, range, and);
            }
            if (null != nested && null != data && data instanceof AbstractIteratorBuilder) {
                AbstractIteratorBuilder iterators = (AbstractIteratorBuilder) data;
                if (negatedLocal) {
                    iterators.addExclude(nested);
                } else {
                    iterators.addInclude(nested);
                }
            } else {
                if (isQueryFullySatisfied == true) {
                    log.warn("Determined that isQueryFullySatisfied should be false, but it was not preset to false in the SatisfactionVisitor");
                }
                return nested;
            }
        }
    } else if (null != data && data instanceof AndIteratorBuilder) {
        and.childrenAccept(this, data);
    } else {
        // Create an AndIterator and recursively add the children
        AbstractIteratorBuilder andItr = new AndIteratorBuilder();
        andItr.negateAsNeeded(data);
        and.childrenAccept(this, andItr);
        // If there is no parent
        if (data == null) {
            // Make this AndIterator the root node
            if (!andItr.includes().isEmpty()) {
                root = andItr.build();
            }
        } else {
            // Otherwise, add this AndIterator to its parent
            AbstractIteratorBuilder parent = (AbstractIteratorBuilder) data;
            if (!andItr.includes().isEmpty()) {
                parent.addInclude(andItr.build());
            }
        }
        if (log.isTraceEnabled()) {
            log.trace("ASTAndNode visit: pretty formatting of:\nparent.includes:" + formatIncludesOrExcludes(andItr.includes()) + "\nparent.excludes:" + formatIncludesOrExcludes(andItr.excludes()));
        }
    }
    return null;
}
Also used : Arrays(java.util.Arrays) Text(org.apache.hadoop.io.Text) ASTSizeMethod(org.apache.commons.jexl2.parser.ASTSizeMethod) AbstractIteratorBuilder(datawave.query.iterator.builder.AbstractIteratorBuilder) JavaRegexParseException(datawave.query.parser.JavaRegexAnalyzer.JavaRegexParseException) IvaratorCacheDir(datawave.query.iterator.ivarator.IvaratorCacheDir) ASTNRNode(org.apache.commons.jexl2.parser.ASTNRNode) Map(java.util.Map) ExceededValueThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode) AttributeFactory(datawave.query.attributes.AttributeFactory) JexlFunctionArgumentDescriptorFactory(datawave.query.jexl.functions.JexlFunctionArgumentDescriptorFactory) ExceededOrThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) TimeFilter(datawave.query.predicate.TimeFilter) Set(java.util.Set) IndexRangeIteratorBuilder(datawave.query.iterator.builder.IndexRangeIteratorBuilder) ASTNENode(org.apache.commons.jexl2.parser.ASTNENode) DefaultArithmetic(datawave.query.jexl.DefaultArithmetic) ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) Predicate(com.google.common.base.Predicate) AndIteratorBuilder(datawave.query.iterator.builder.AndIteratorBuilder) JexlContext(org.apache.commons.jexl2.JexlContext) DatawaveFieldIndexListIteratorJexl(datawave.core.iterators.DatawaveFieldIndexListIteratorJexl) TermFrequencyAggregator(datawave.query.jexl.functions.TermFrequencyAggregator) FileSortedSet(datawave.query.util.sortedset.FileSortedSet) IteratorBuilder(datawave.query.iterator.builder.IteratorBuilder) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) NestedIterator(datawave.query.iterator.NestedIterator) JexlASTHelper(datawave.query.jexl.JexlASTHelper) IndexIteratorBuilder(datawave.query.iterator.builder.IndexIteratorBuilder) JexlArithmetic(org.apache.commons.jexl2.JexlArithmetic) ASTMethodNode(org.apache.commons.jexl2.parser.ASTMethodNode) IteratorToSortedKeyValueIterator(datawave.query.util.IteratorToSortedKeyValueIterator) JexlNode(org.apache.commons.jexl2.parser.JexlNode) SortedKeyValueIterator(org.apache.accumulo.core.iterators.SortedKeyValueIterator) TreeSet(java.util.TreeSet) SourceFactory(datawave.query.iterator.SourceFactory) ArrayList(java.util.ArrayList) NodeOperand(datawave.query.jexl.LiteralRange.NodeOperand) Lists(com.google.common.collect.Lists) LiteralRange(datawave.query.jexl.LiteralRange) Key(org.apache.accumulo.core.data.Key) IndexFilterIteratorBuilder(datawave.query.iterator.builder.IndexFilterIteratorBuilder) OrIteratorBuilder(datawave.query.iterator.builder.OrIteratorBuilder) Predicates(com.google.common.base.Predicates) EventFieldIterator(datawave.query.iterator.EventFieldIterator) IndexRegexIteratorBuilder(datawave.query.iterator.builder.IndexRegexIteratorBuilder) FileSystemCache(datawave.core.iterators.filesystem.FileSystemCache) FieldIndexAggregator(datawave.query.jexl.functions.FieldIndexAggregator) QueryPropertyMarker(datawave.query.jexl.nodes.QueryPropertyMarker) TermFrequencyIndexBuilder(datawave.query.iterator.builder.TermFrequencyIndexBuilder) IOException(java.io.IOException) IndexListIteratorBuilder(datawave.query.iterator.builder.IndexListIteratorBuilder) DatawaveErrorCode(datawave.webservice.query.exception.DatawaveErrorCode) QueryLock(datawave.core.iterators.querylock.QueryLock) OrIterator(datawave.query.iterator.logic.OrIterator) NegationBuilder(datawave.query.iterator.builder.NegationBuilder) Range(org.apache.accumulo.core.data.Range) JavaRegexAnalyzer(datawave.query.parser.JavaRegexAnalyzer) Filter(datawave.query.predicate.Filter) QueryException(datawave.webservice.query.exception.QueryException) ASTEvaluationOnly(org.apache.commons.jexl2.parser.ASTEvaluationOnly) FST(org.apache.lucene.util.fst.FST) PartialKey(org.apache.accumulo.core.data.PartialKey) QuerySpanCollector(datawave.query.iterator.profile.QuerySpanCollector) IvaratorBuilder(datawave.query.iterator.builder.IvaratorBuilder) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) IteratorEnvironment(org.apache.accumulo.core.iterators.IteratorEnvironment) Script(org.apache.commons.jexl2.Script) SortedSet(java.util.SortedSet) DatawaveJexlEngine(datawave.query.jexl.DatawaveJexlEngine) URISyntaxException(java.net.URISyntaxException) SourceManager(datawave.query.iterator.SourceManager) Logger(org.apache.log4j.Logger) ValueTuple(datawave.query.attributes.ValueTuple) EventDataQueryFilter(datawave.query.predicate.EventDataQueryFilter) ASTNotNode(org.apache.commons.jexl2.parser.ASTNotNode) ASTStringLiteral(org.apache.commons.jexl2.parser.ASTStringLiteral) Path(org.apache.hadoop.fs.Path) Value(org.apache.accumulo.core.data.Value) URI(java.net.URI) UniversalSet(datawave.util.UniversalSet) TypeMetadata(datawave.query.util.TypeMetadata) ASTDelayedPredicate(org.apache.commons.jexl2.parser.ASTDelayedPredicate) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) List(java.util.List) IdentifierOpLiteral(datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral) Entry(java.util.Map.Entry) IvaratorCacheDirConfig(datawave.query.iterator.ivarator.IvaratorCacheDirConfig) EventDataQueryExpressionFilter(datawave.query.predicate.EventDataQueryExpressionFilter) JexlNodes.children(org.apache.commons.jexl2.parser.JexlNodes.children) GenericObjectPool(org.apache.commons.pool.impl.GenericObjectPool) ASTReferenceExpression(org.apache.commons.jexl2.parser.ASTReferenceExpression) ASTNumberLiteral(org.apache.commons.jexl2.parser.ASTNumberLiteral) NoOpType(datawave.data.type.NoOpType) IdentityAggregator(datawave.query.jexl.functions.IdentityAggregator) MessageFormat(java.text.MessageFormat) HashSet(java.util.HashSet) ASTERNode(org.apache.commons.jexl2.parser.ASTERNode) DatawaveJexlContext(datawave.query.jexl.DatawaveJexlContext) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode) NoSuchElementException(java.util.NoSuchElementException) ParserTreeConstants(org.apache.commons.jexl2.parser.ParserTreeConstants) MalformedURLException(java.net.MalformedURLException) Maps(com.google.common.collect.Maps) Constants(datawave.query.Constants) ArithmeticJexlEngines(datawave.query.jexl.ArithmeticJexlEngines) ASTFunctionNode(org.apache.commons.jexl2.parser.ASTFunctionNode) CompositeMetadata(datawave.query.composite.CompositeMetadata) EventFieldAggregator(datawave.query.jexl.functions.EventFieldAggregator) Collections(java.util.Collections) ChainableEventDataQueryFilter(datawave.query.predicate.ChainableEventDataQueryFilter) ASTReference(org.apache.commons.jexl2.parser.ASTReference) ASTNotNode(org.apache.commons.jexl2.parser.ASTNotNode) ASTERNode(org.apache.commons.jexl2.parser.ASTERNode) ASTNRNode(org.apache.commons.jexl2.parser.ASTNRNode) QueryPropertyMarker(datawave.query.jexl.nodes.QueryPropertyMarker) IOException(java.io.IOException) AndIteratorBuilder(datawave.query.iterator.builder.AndIteratorBuilder) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) QueryException(datawave.webservice.query.exception.QueryException) ASTFunctionNode(org.apache.commons.jexl2.parser.ASTFunctionNode) AbstractIteratorBuilder(datawave.query.iterator.builder.AbstractIteratorBuilder) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) ExceededValueThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode) ExceededOrThresholdMarkerJexlNode(datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode) JexlNode(org.apache.commons.jexl2.parser.JexlNode) IndexRangeIteratorBuilder(datawave.query.iterator.builder.IndexRangeIteratorBuilder) LiteralRange(datawave.query.jexl.LiteralRange) Key(org.apache.accumulo.core.data.Key) PartialKey(org.apache.accumulo.core.data.PartialKey) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode)

Example 5 with ASTAndNode

use of org.apache.commons.jexl2.parser.ASTAndNode in project datawave by NationalSecurityAgency.

the class JexlASTHelperTest method validateJunctionChildrenWithInvalidTree.

// Verify that false is returned for a query with invalid junctions when failHard == false.
@Test
public void validateJunctionChildrenWithInvalidTree() {
    ASTEQNode eqNode = (ASTEQNode) JexlNodeFactory.buildEQNode("FOO", "bar");
    ASTAndNode conjunction = (ASTAndNode) JexlNodeFactory.createAndNode(Collections.singletonList(eqNode));
    assertFalse(JexlASTHelper.validateJunctionChildren(conjunction));
    assertFalse(JexlASTHelper.validateJunctionChildren(conjunction, false));
}
Also used : ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode) Test(org.junit.Test)

Aggregations

JexlNode (org.apache.commons.jexl2.parser.JexlNode)48 ASTAndNode (org.apache.commons.jexl2.parser.ASTAndNode)41 ASTOrNode (org.apache.commons.jexl2.parser.ASTOrNode)23 ExceededValueThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode)17 ArrayList (java.util.ArrayList)14 ASTEQNode (org.apache.commons.jexl2.parser.ASTEQNode)11 ExceededTermThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededTermThresholdMarkerJexlNode)9 ASTReferenceExpression (org.apache.commons.jexl2.parser.ASTReferenceExpression)9 QueryPropertyMarker (datawave.query.jexl.nodes.QueryPropertyMarker)8 ASTReference (org.apache.commons.jexl2.parser.ASTReference)8 LiteralRange (datawave.query.jexl.LiteralRange)7 List (java.util.List)6 ASTIdentifier (org.apache.commons.jexl2.parser.ASTIdentifier)6 LinkedList (java.util.LinkedList)5 ASTFunctionNode (org.apache.commons.jexl2.parser.ASTFunctionNode)5 DatawaveFatalQueryException (datawave.query.exceptions.DatawaveFatalQueryException)4 ExceededOrThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededOrThresholdMarkerJexlNode)4 QueryException (datawave.webservice.query.exception.QueryException)4 Collection (java.util.Collection)4 ASTGENode (org.apache.commons.jexl2.parser.ASTGENode)4