Search in sources :

Example 1 with ASTReference

use of org.apache.commons.jexl2.parser.ASTReference in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitor method canExpand.

/**
 * Test if the current node should be expanded by this visitor. Update the tracker with the current andChild if expansion is possible
 *
 * @param node
 *            the node to test
 * @param tracker
 *            the tracker holding supplementary information about the expansion
 * @return true if the expansion should occur, false otherwise
 */
private boolean canExpand(JexlNode node, ExecutableDeterminationVisitor.STATE state, ExpansionTracker tracker) {
    // only process if state is ERROR or PARTIAL
    if (!(state == ExecutableDeterminationVisitor.STATE.ERROR || state == ExecutableDeterminationVisitor.STATE.PARTIAL)) {
        return false;
    }
    // if deeper down in the tree there was a failed expansion attempt, don't try again
    if (tracker.isFailedExpansion()) {
        return false;
    }
    // there must be an andNode further up the tree to distribute into the or
    ASTAndNode lastAnd = tracker.getLastAnd();
    // as long as there is a lastAnd there is work to do
    if (lastAnd == null) {
        return false;
    }
    // verify there is nothing but compatible nodes between the andNode and the orNode
    JexlNode current = node.jjtGetParent();
    JexlNode last = node;
    while (current != lastAnd) {
        if (!(current instanceof ASTReference || current instanceof ASTReferenceExpression)) {
            return false;
        }
        last = current;
        current = current.jjtGetParent();
    }
    // if we got here the expansion is good and we should track the andChild for later use
    tracker.setAndChild(last);
    return true;
}
Also used : ASTReferenceExpression(org.apache.commons.jexl2.parser.ASTReferenceExpression) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ASTReference(org.apache.commons.jexl2.parser.ASTReference) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode)

Example 2 with ASTReference

use of org.apache.commons.jexl2.parser.ASTReference in project datawave by NationalSecurityAgency.

the class DistributeAndedNodesVisitor method visit.

/**
 * Checks each of the child nodes, and determines how the anded nodes should be applied.
 *
 * @param node
 *            The node that we will be distributing the anded nodes into
 * @param data
 *            The nodes which we will be distributing into the root node
 * @return An updated script with the anded nodes distributed throughout
 */
@Override
public Object visit(ASTOrNode node, Object data) {
    DistributeAndedNodesVisitor.DistAndData parentData = (DistributeAndedNodesVisitor.DistAndData) data;
    if (initialNode == null || initialNode instanceof ASTReference || initialNode instanceof ASTReferenceExpression)
        initialNode = node;
    // if this node is one of the anded nodes, or a whindex
    // comprised of one of the anded nodes, halt recursion
    List<JexlNode> usedAndedNodes = usesAndedNodes(node);
    if (!usedAndedNodes.isEmpty()) {
        parentData.usedAndedNodes.addAll(usedAndedNodes);
        return node;
    }
    // this logic is dependent upon identifying whindex nodes by their address
    if (whindexNodes.containsKey(node)) {
        return node;
    }
    boolean rebuildNode = false;
    // check each child node
    List<JexlNode> nodesMissingEverything = new ArrayList<>();
    List<JexlNode> nodesWithEverything = new ArrayList<>();
    Map<JexlNode, List<JexlNode>> nodesMissingSomething = new LinkedHashMap<>();
    for (JexlNode child : JexlNodes.children(node)) {
        DistributeAndedNodesVisitor.DistAndData foundData = new DistributeAndedNodesVisitor.DistAndData();
        JexlNode processedChild = (JexlNode) child.jjtAccept(this, foundData);
        if (processedChild != child)
            rebuildNode = true;
        if (foundData.usedAndedNodes.isEmpty())
            nodesMissingEverything.add(processedChild);
        else if (!foundData.usedAndedNodes.containsAll(andedNodes)) {
            List<JexlNode> missingAndedNodes = new ArrayList<>(andedNodes);
            missingAndedNodes.removeAll(foundData.usedAndedNodes);
            nodesMissingSomething.put(processedChild, missingAndedNodes);
        } else
            nodesWithEverything.add(processedChild);
    }
    // if none of the children are missing anything, we're done
    if (nodesWithEverything.size() == node.jjtGetNumChildren()) {
        parentData.usedAndedNodes.addAll(andedNodes);
        if (rebuildNode)
            return WhindexVisitor.createUnwrappedOrNode(nodesWithEverything);
        else
            return node;
    } else // are missing everything, it implies that the children were left as-is
    if (nodesMissingEverything.size() == node.jjtGetNumChildren()) {
        return node;
    }
    // if we got here, then there are some nodes missing SOMETHING, and we have work to do
    List<JexlNode> rebuiltChildren = new ArrayList<>();
    // andedNodes
    for (Map.Entry<JexlNode, List<JexlNode>> childEntry : nodesMissingSomething.entrySet()) rebuiltChildren.add(DistributeAndedNodesVisitor.distributeAndedNode(childEntry.getKey(), childEntry.getValue(), whindexNodes));
    // for children missing everything -> 'or' them together, then 'and' them with the full set of andedNodes
    List<JexlNode> nodeList = andedNodes.stream().map(RebuildingVisitor::copy).collect(Collectors.toList());
    if (!nodesMissingEverything.isEmpty()) {
        nodeList.add(WhindexVisitor.createUnwrappedOrNode(nodesMissingEverything));
    }
    rebuiltChildren.add(WhindexVisitor.createUnwrappedAndNode(nodeList));
    // for children with everything -> keep those as-is
    rebuiltChildren.addAll(nodesWithEverything);
    parentData.usedAndedNodes.addAll(andedNodes);
    return WhindexVisitor.createUnwrappedOrNode(rebuiltChildren);
}
Also used : ASTReferenceExpression(org.apache.commons.jexl2.parser.ASTReferenceExpression) ArrayList(java.util.ArrayList) ASTReference(org.apache.commons.jexl2.parser.ASTReference) LinkedHashMap(java.util.LinkedHashMap) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ArrayList(java.util.ArrayList) List(java.util.List) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 3 with ASTReference

use of org.apache.commons.jexl2.parser.ASTReference in project datawave by NationalSecurityAgency.

the class QueryModelVisitor method expandBinaryNodeFromModel.

/**
 * Applies the forward mapping from the QueryModel to a node, expanding the node into an Or if needed.
 *
 * @param node
 * @param data
 * @return
 */
protected JexlNode expandBinaryNodeFromModel(JexlNode node, Object data) {
    String field = JexlASTHelper.getIdentifier(node);
    if (isFieldExcluded(field)) {
        return node;
    }
    // Count the immediate children:
    int childCount = node.jjtGetNumChildren();
    if (childCount != 2) {
        QueryException qe = new QueryException(DatawaveErrorCode.BINARY_NODE_TOO_MANY_CHILDREN, MessageFormat.format("Node: {0}", PrintingVisitor.formattedQueryString(node)));
        throw new DatawaveFatalQueryException(qe);
    }
    // Find identifiers
    List<ASTIdentifier> allidentifiers = JexlASTHelper.getIdentifiers(node);
    // If we don't have any identifiers, we have nothing to expand
    if (allidentifiers.isEmpty()) {
        return node;
    }
    JexlNode leftNode = node.jjtGetChild(0);
    JexlNode rightNode = node.jjtGetChild(1);
    if (log.isTraceEnabled()) {
        log.trace("leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // this will expand identifiers that have a method connected to them
    boolean leftState = JexlASTHelper.HasMethodVisitor.hasMethod(leftNode);
    if (leftState) {
        // there is a method under leftNode
        leftNode = (JexlNode) leftNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    boolean rightState = JexlASTHelper.HasMethodVisitor.hasMethod(rightNode);
    if (rightState) {
        // there is a method under rightNode
        rightNode = (JexlNode) rightNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    // expand any identifiers inside of methods/functions in the left and right nodes
    leftNode = (JexlNode) leftNode.jjtAccept(this, null);
    rightNode = (JexlNode) rightNode.jjtAccept(this, null);
    if (log.isTraceEnabled()) {
        log.trace("after expansion, leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("after expansion, leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("after expansion, rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("after expansion, rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // if state == true on either side, then there is a method on one side and we are done applying the model
    if (leftState || rightState) {
        JexlNode toReturn = JexlNodeFactory.buildUntypedBinaryNode(node, leftNode, rightNode);
        if (log.isTraceEnabled()) {
            log.trace("done early. returning:" + JexlStringBuildingVisitor.buildQuery(toReturn));
        }
        return toReturn;
    }
    JexlNode leftSeed, rightSeed;
    Set<JexlNode> left = Sets.newHashSet(), right = Sets.newHashSet();
    boolean isNullEquality = false;
    if (node instanceof ASTEQNode && (leftNode instanceof ASTNullLiteral || rightNode instanceof ASTNullLiteral)) {
        isNullEquality = true;
    }
    // the query has been previously groomed so that identifiers are on the left and literals are on the right
    // an identifier with a method attached will have already been substituted above (and will return null for the IdentifierOpLiteral)
    // The normal case of `IDENTIFIER op 'literal'`
    JexlASTHelper.IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op != null) {
        // One identifier
        leftSeed = op.getIdentifier();
        rightSeed = op.getLiteral();
        if (rightSeed instanceof ASTNullLiteral && node instanceof ASTEQNode) {
            isNullEquality = true;
        }
    } else {
        // We know from above that childCount == 2. We may have a reference on both sides of the expression
        leftSeed = node.jjtGetChild(0);
        rightSeed = node.jjtGetChild(1);
    }
    if (leftSeed instanceof ASTReference) {
        // String fieldName = JexlASTHelper.getIdentifier((JexlNode)leftSeed);
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(leftSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(leftSeed) + " from " + JexlStringBuildingVisitor.buildQuery(leftNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                left.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (leftSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) leftSeed))) {
            left.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        left.add(leftSeed);
    }
    if (rightSeed instanceof ASTReference) {
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(rightSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(rightSeed) + " from " + JexlStringBuildingVisitor.buildQuery(rightNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                right.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (rightSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) rightSeed))) {
            right.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        right.add(rightSeed);
    }
    boolean requiresAnd = isNullEquality || node instanceof ASTNENode;
    @SuppressWarnings("unchecked") Set<List<JexlNode>> // retrieve the cartesian product
    product = Sets.cartesianProduct(left, right);
    /**
     * use the product transformer to shallow copy the jexl nodes. We've created new nodes that will be embedded within an ast reference. As a result, we
     * need to ensure that if we create a logical structure ( such as an or ) -- each literal references a unique identifier from the right. Otherwise,
     * subsequent visitors will reference incorrection sub trees, and potentially negate the activity of the query model visitor
     */
    Set<List<JexlNode>> newSet = product.stream().map(list -> list.stream().map(RebuildingVisitor::copy).collect(Collectors.toList())).collect(Collectors.toSet());
    if (product.size() > 1) {
        JexlNode expanded;
        if (requiresAnd) {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.AND_NODE, node, newSet);
        } else {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.OR_NODE, node, newSet);
        }
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    } else if (1 == product.size()) {
        List<JexlNode> pair = product.iterator().next();
        JexlNode expanded = JexlNodeFactory.buildUntypedBinaryNode(node, pair.get(0), pair.get(1));
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    }
    // If we couldn't map anything, return a copy
    if (log.isTraceEnabled())
        log.trace("just returning the original:" + PrintingVisitor.formattedQueryString(node));
    return node;
}
Also used : ContainerType(datawave.query.jexl.JexlNodeFactory.ContainerType) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) ASTLTNode(org.apache.commons.jexl2.parser.ASTLTNode) JexlASTHelper(datawave.query.jexl.JexlASTHelper) ASTLENode(org.apache.commons.jexl2.parser.ASTLENode) ASTMethodNode(org.apache.commons.jexl2.parser.ASTMethodNode) QueryModel(datawave.query.model.QueryModel) JexlNode(org.apache.commons.jexl2.parser.JexlNode) JexlNodeFactory(datawave.query.jexl.JexlNodeFactory) ASTSizeMethod(org.apache.commons.jexl2.parser.ASTSizeMethod) MessageFormat(java.text.MessageFormat) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ASTERNode(org.apache.commons.jexl2.parser.ASTERNode) Logger(org.apache.log4j.Logger) Lists(com.google.common.collect.Lists) ASTNRNode(org.apache.commons.jexl2.parser.ASTNRNode) LiteralRange(datawave.query.jexl.LiteralRange) ASTGTNode(org.apache.commons.jexl2.parser.ASTGTNode) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode) ThreadConfigurableLogger(datawave.webservice.common.logging.ThreadConfigurableLogger) ParserTreeConstants(org.apache.commons.jexl2.parser.ParserTreeConstants) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) ASTNullLiteral(org.apache.commons.jexl2.parser.ASTNullLiteral) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) JexlNodes(org.apache.commons.jexl2.parser.JexlNodes) BoundedRange(datawave.query.jexl.nodes.BoundedRange) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) DatawaveErrorCode(datawave.webservice.query.exception.DatawaveErrorCode) Node(org.apache.commons.jexl2.parser.Node) ASTNENode(org.apache.commons.jexl2.parser.ASTNENode) ASTGENode(org.apache.commons.jexl2.parser.ASTGENode) List(java.util.List) QueryException(datawave.webservice.query.exception.QueryException) ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) ASTFunctionNode(org.apache.commons.jexl2.parser.ASTFunctionNode) Collections(java.util.Collections) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) ASTReference(org.apache.commons.jexl2.parser.ASTReference) ASTNENode(org.apache.commons.jexl2.parser.ASTNENode) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) ASTNullLiteral(org.apache.commons.jexl2.parser.ASTNullLiteral) ASTReference(org.apache.commons.jexl2.parser.ASTReference) JexlASTHelper(datawave.query.jexl.JexlASTHelper) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) QueryException(datawave.webservice.query.exception.QueryException) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ArrayList(java.util.ArrayList) List(java.util.List)

Example 4 with ASTReference

use of org.apache.commons.jexl2.parser.ASTReference in project datawave by NationalSecurityAgency.

the class AncestorUidIntersectorTest method setup.

@Before
public void setup() {
    intersector = new AncestorUidIntersector();
    uids1 = new TreeSet<>();
    uids2 = new TreeSet<>();
    node1 = new ASTEQNode(1);
    JexlNode reference1 = new ASTReference(2);
    node1.jjtAddChild(reference1, 0);
    JexlNode name = new ASTIdentifier(3);
    name.image = "fieldName1";
    reference1.jjtAddChild(name, 0);
    JexlNode reference2 = new ASTReference(4);
    node1.jjtAddChild(reference2, 1);
    JexlNode value = new ASTStringLiteral(5);
    value.image = "fieldValue1";
    reference2.jjtAddChild(value, 0);
    node2 = new ASTEQNode(6);
    reference1 = new ASTReference(7);
    node2.jjtAddChild(reference1, 0);
    name = new ASTIdentifier(8);
    name.image = "fieldName2";
    reference1.jjtAddChild(name, 0);
    reference2 = new ASTReference(9);
    node2.jjtAddChild(reference2, 1);
    value = new ASTStringLiteral(10);
    value.image = "fieldValue2";
    reference2.jjtAddChild(value, 0);
}
Also used : ASTStringLiteral(org.apache.commons.jexl2.parser.ASTStringLiteral) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) ASTReference(org.apache.commons.jexl2.parser.ASTReference) Before(org.junit.Before)

Example 5 with ASTReference

use of org.apache.commons.jexl2.parser.ASTReference in project datawave by NationalSecurityAgency.

the class JexlASTHelper method normalizeLiteral.

public static ASTReference normalizeLiteral(JexlNode literal, Type<?> normalizer) throws NormalizationException {
    String normalizedImg = normalizer.normalize(literal.image);
    ASTStringLiteral normalizedLiteral = new ASTStringLiteral(ParserTreeConstants.JJTSTRINGLITERAL);
    normalizedLiteral.image = normalizedImg;
    return JexlNodes.makeRef(normalizedLiteral);
}
Also used : ASTStringLiteral(org.apache.commons.jexl2.parser.ASTStringLiteral)

Aggregations

ASTReference (org.apache.commons.jexl2.parser.ASTReference)28 JexlNode (org.apache.commons.jexl2.parser.JexlNode)19 ASTReferenceExpression (org.apache.commons.jexl2.parser.ASTReferenceExpression)11 ASTIdentifier (org.apache.commons.jexl2.parser.ASTIdentifier)8 ASTAndNode (org.apache.commons.jexl2.parser.ASTAndNode)7 ASTStringLiteral (org.apache.commons.jexl2.parser.ASTStringLiteral)7 ASTOrNode (org.apache.commons.jexl2.parser.ASTOrNode)6 ASTEQNode (org.apache.commons.jexl2.parser.ASTEQNode)5 ArrayList (java.util.ArrayList)4 ASTERNode (org.apache.commons.jexl2.parser.ASTERNode)4 ASTFunctionNode (org.apache.commons.jexl2.parser.ASTFunctionNode)4 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)4 ASTNRNode (org.apache.commons.jexl2.parser.ASTNRNode)4 ExceededValueThresholdMarkerJexlNode (datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode)3 List (java.util.List)3 ASTAssignment (org.apache.commons.jexl2.parser.ASTAssignment)3 ASTNumberLiteral (org.apache.commons.jexl2.parser.ASTNumberLiteral)3 ASTIdentifier (org.apache.commons.jexl3.parser.ASTIdentifier)3 ASTReference (org.apache.commons.jexl3.parser.ASTReference)3 DatawaveFatalQueryException (datawave.query.exceptions.DatawaveFatalQueryException)2