Examples with ExceededValueThresholdMarkerJexlNode datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode used on opensource projects

Example 1 with ExceededValueThresholdMarkerJexlNode

use of datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitor method visit.

@Override
public Object visit(ASTAndNode and, Object data) {
    QueryPropertyMarker.Instance instance = QueryPropertyMarker.findInstance(and);
    if (instance.isType(ExceededOrThresholdMarkerJexlNode.class)) {
        JexlNode source = instance.getSource();
        // Ivarator to get the job done
        if (source instanceof ASTAndNode) {
            try {
                ivarateList(and, source, data);
            } catch (IOException ioe) {
                throw new DatawaveFatalQueryException(ioe);
            }
        } else {
            QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "Limited ExceededOrThresholdMarkerJexlNode"));
            throw new DatawaveFatalQueryException(qe);
        }
    } else if (data instanceof IndexRangeIteratorBuilder) {
        // index checking has already been done, otherwise we would not have
        // an "ExceededValueThresholdMarker"
        // hence the "IndexAgnostic" method can be used here
        LiteralRange range = JexlASTHelper.findRange().recursively().getRange(and);
        if (range == null) {
            QueryException qe = new QueryException(DatawaveErrorCode.MULTIPLE_RANGES_IN_EXPRESSION);
            throw new DatawaveFatalQueryException(qe);
        }
        ((IndexRangeIteratorBuilder) data).setRange(range);
    } else if (instance.isType(ExceededValueThresholdMarkerJexlNode.class)) {
        // if the parent is our ExceededValueThreshold marker, then use an
        // Ivarator to get the job done unless we don't have to
        JexlNode source = instance.getSource();
        String identifier = null;
        LiteralRange<?> range = null;
        boolean negatedLocal = false;
        if (source instanceof ASTAndNode) {
            range = buildLiteralRange(source, null);
            identifier = range.getFieldName();
        } else {
            if (source instanceof ASTNRNode || source instanceof ASTNotNode)
                negatedLocal = true;
            range = buildLiteralRange(source);
            identifier = JexlASTHelper.getIdentifier(source);
        }
        boolean negatedOverall = negatedLocal;
        if (data instanceof AbstractIteratorBuilder) {
            AbstractIteratorBuilder oib = (AbstractIteratorBuilder) data;
            if (oib.isInANot()) {
                negatedOverall = !negatedOverall;
            }
        }
        // or the field is index only but not in the term frequencies, then we must ivarate
        if (!limitLookup || !allowTermFrequencyLookup || (indexOnlyFields.contains(identifier) && !termFrequencyFields.contains(identifier))) {
            if (source instanceof ASTAndNode) {
                try {
                    List<ASTFunctionNode> functionNodes = JexlASTHelper.getFunctionNodes(source).stream().filter(node -> JexlFunctionArgumentDescriptorFactory.F.getArgumentDescriptor(node).allowIvaratorFiltering()).collect(Collectors.toList());
                    if (functionNodes.isEmpty()) {
                        ivarateRange(and, source, data);
                    } else {
                        ivarateFilter(and, source, data, functionNodes);
                    }
                } catch (IOException ioe) {
                    throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
                }
            } else if (source instanceof ASTERNode || source instanceof ASTNRNode) {
                try {
                    ivarateRegex(and, source, data);
                } catch (IOException ioe) {
                    throw new DatawaveFatalQueryException("Unable to ivarate", ioe);
                }
            } else {
                QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "ExceededValueThresholdMarkerJexlNode"));
                throw new DatawaveFatalQueryException(qe);
            }
        } else {
            NestedIterator<Key> nested = null;
            if (termFrequencyFields.contains(identifier)) {
                nested = buildExceededFromTermFrequency(identifier, and, source, range, data);
            } else {
                /**
                 * This is okay since 1) We are doc specific 2) We are not index only or tf 3) Therefore, we must evaluate against the document for this
                 * expression 4) Return a stubbed range in case we have a disjunction that breaks the current doc.
                 */
                if (!limitOverride && !negatedOverall)
                    nested = createExceededCheck(identifier, range, and);
            }
            if (null != nested && null != data && data instanceof AbstractIteratorBuilder) {
                AbstractIteratorBuilder iterators = (AbstractIteratorBuilder) data;
                if (negatedLocal) {
                    iterators.addExclude(nested);
                } else {
                    iterators.addInclude(nested);
                }
            } else {
                if (isQueryFullySatisfied == true) {
                    log.warn("Determined that isQueryFullySatisfied should be false, but it was not preset to false in the SatisfactionVisitor");
                }
                return nested;
            }
        }
    } else if (null != data && data instanceof AndIteratorBuilder) {
        and.childrenAccept(this, data);
    } else {
        // Create an AndIterator and recursively add the children
        AbstractIteratorBuilder andItr = new AndIteratorBuilder();
        andItr.negateAsNeeded(data);
        and.childrenAccept(this, andItr);
        // If there is no parent
        if (data == null) {
            // Make this AndIterator the root node
            if (!andItr.includes().isEmpty()) {
                root = andItr.build();
            }
        } else {
            // Otherwise, add this AndIterator to its parent
            AbstractIteratorBuilder parent = (AbstractIteratorBuilder) data;
            if (!andItr.includes().isEmpty()) {
                parent.addInclude(andItr.build());
            }
        }
        if (log.isTraceEnabled()) {
            log.trace("ASTAndNode visit: pretty formatting of:\nparent.includes:" + formatIncludesOrExcludes(andItr.includes()) + "\nparent.excludes:" + formatIncludesOrExcludes(andItr.excludes()));
        }
    }
    return null;
}

Example 2 with ExceededValueThresholdMarkerJexlNode

use of datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testExceededThresholdExpansionExternal.

@Test
public void testExceededThresholdExpansionExternal() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    // update the generated queryTree to have an ExceededThreshold marker
    JexlNode child = new ExceededValueThresholdMarkerJexlNode(queryTree.jjtGetChild(0).jjtGetChild(0));
    // unlink the old node
    queryTree.jjtGetChild(0).jjtGetChild(0).jjtSetParent(null);
    // overwrite the old UUID==capone with the ExceededThreshold marker
    queryTree.jjtGetChild(0).jjtAddChild(child, 0);
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // included ExceededValueThresholdMarker before
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(queryTree), JexlStringBuildingVisitor.buildQuery(queryTree).equals("((_Value_ = true) && (UUID == 'capone')) && (filter:includeRegex(QUOTE, '.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')"));
    // not executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // it looks like what we'd expect
    String expected = "(QUOTE == 'kind' && ((_Value_ = true) && (UUID == 'capone'))) || " + "((filter:includeRegex(QUOTE, '.*kind.*') || BIRTH_DATE == '123') && ((_Value_ = true) && (UUID == 'capone')))";
    Assert.assertEquals(expected, JexlStringBuildingVisitor.buildQueryWithoutParse(newTree));
}

Example 3 with ExceededValueThresholdMarkerJexlNode

use of datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode in project datawave by NationalSecurityAgency.

the class ExecutableExpansionVisitorTest method testExceededThresholdExpansionInternal.

@Test
public void testExceededThresholdExpansionInternal() throws Exception {
    ASTJexlScript queryTree = JexlASTHelper.parseJexlQuery("UUID == 'capone' && (filter:includeRegex(QUOTE,'.*kind.*') || QUOTE == 'kind' || BIRTH_DATE == '123')");
    // update the generated queryTree to have an ExceededThreshold marker for BIRTH_DATE
    JexlNode child = new ExceededValueThresholdMarkerJexlNode(queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0).jjtGetChild(0).jjtGetChild(1));
    // unlink the old node
    queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0).jjtGetChild(0).jjtGetChild(1).jjtSetParent(null);
    // overwrite the old BIRTH_DATE==123 with the ExceededThreshold marker
    queryTree.jjtGetChild(0).jjtGetChild(1).jjtGetChild(0).jjtGetChild(0).jjtAddChild(child, 1);
    ShardQueryConfiguration config = EasyMock.createMock(ShardQueryConfiguration.class);
    MetadataHelper helper = EasyMock.createMock(MetadataHelper.class);
    HashSet<String> indexedFields = new HashSet<>();
    indexedFields.add("UUID");
    indexedFields.add("QUOTE");
    EasyMock.expect(config.getIndexedFields()).andReturn(indexedFields).anyTimes();
    EasyMock.replay(config, helper);
    ASTJexlScript newTree = ExecutableExpansionVisitor.expand(queryTree, config, helper);
    EasyMock.verify(config, helper);
    // included ExceededValueThresholdMarker before
    Assert.assertTrue(JexlStringBuildingVisitor.buildQuery(queryTree), JexlStringBuildingVisitor.buildQuery(queryTree).equals("UUID == 'capone' && (filter:includeRegex(QUOTE, '.*kind.*') || QUOTE == 'kind' || " + "((_Value_ = true) && (BIRTH_DATE == '123')))"));
    // not executable
    Assert.assertFalse(ExecutableDeterminationVisitor.isExecutable(queryTree, config, helper));
    // what came out is executable
    Assert.assertTrue(ExecutableDeterminationVisitor.isExecutable(newTree, config, helper));
    // it looks like what we'd expect
    String expected = "(QUOTE == 'kind' && UUID == 'capone') || " + "(((_Value_ = true) && (BIRTH_DATE == '123')) && UUID == 'capone') || " + "(filter:includeRegex(QUOTE, '.*kind.*') && UUID == 'capone')";
    Assert.assertEquals(expected, JexlStringBuildingVisitor.buildQueryWithoutParse(newTree));
}

Example 4 with ExceededValueThresholdMarkerJexlNode

use of datawave.query.jexl.nodes.ExceededValueThresholdMarkerJexlNode in project datawave by NationalSecurityAgency.

the class JexlNodeFactory method createNodeTreeFromFieldsToValues.

/**
 * Expand a node given a mapping of fields to values. If the list is empty, then the original regex should be used.
 *
 * @param containerType
 *            should we create OR nodes or AND nodes
 * @param node
 * @param fieldsToValues
 *            A mapping of fields to values. If the values for a field is empty, then the original regex should be used.
 * @param expandFields
 *            Expand fields if true
 * @param expandValues
 *            Expand values if true
 * @param keepOriginalNode
 *            Keep the original node along with any expansions
 * @return A new sub query
 */
public static JexlNode createNodeTreeFromFieldsToValues(ContainerType containerType, JexlNode node, JexlNode orgNode, IndexLookupMap fieldsToValues, boolean expandFields, boolean expandValues, boolean keepOriginalNode) {
    // do nothing if not expanding fields or values
    if (!expandFields && !expandValues) {
        return orgNode;
    }
    // no expansions needed if the fieldname threshold is exceeded
    if (fieldsToValues.isKeyThresholdExceeded()) {
        return new ExceededTermThresholdMarkerJexlNode(orgNode);
    }
    // collapse the value sets if not expanding fields
    if (!expandFields) {
        ValueSet allValues = new ValueSet(-1);
        for (ValueSet values : fieldsToValues.values()) {
            allValues.addAll(values);
        }
        fieldsToValues.clear();
        for (String identifier : JexlASTHelper.getIdentifierNames(orgNode)) {
            fieldsToValues.put(identifier, allValues);
        }
    }
    Set<String> fields = fieldsToValues.keySet();
    JexlNode parentNode = (containerType.equals(ContainerType.OR_NODE) ? new ASTOrNode(ParserTreeConstants.JJTORNODE) : new ASTAndNode(ParserTreeConstants.JJTANDNODE));
    int parentNodeChildCount = 0;
    if (keepOriginalNode) {
        JexlNodes.ensureCapacity(parentNode, fields.size() + 1);
        JexlNode child = RebuildingVisitor.copy(orgNode);
        parentNode.jjtAddChild(child, parentNodeChildCount);
        child.jjtSetParent(parentNode);
        parentNodeChildCount++;
        // remove this entry from the fieldsToValues to avoid duplication
        for (String identifier : JexlASTHelper.getIdentifierNames(orgNode)) {
            for (Object value : JexlASTHelper.getLiteralValues(orgNode)) {
                fieldsToValues.remove(identifier, value);
            }
        }
    } else {
        JexlNodes.ensureCapacity(parentNode, fields.size());
    }
    for (String field : fields) {
        ValueSet valuesForField = fieldsToValues.get(field);
        // if not expanding values, then reuse the original node with simply a new field name (anyfield only)
        if (!expandValues) {
            JexlNode child = RebuildingVisitor.copy(orgNode);
            for (ASTIdentifier identifier : JexlASTHelper.getIdentifiers(child)) {
                if (identifier.image.equals(Constants.ANY_FIELD)) {
                    identifier.image = field;
                }
            }
            parentNode.jjtAddChild(child, parentNodeChildCount);
            child.jjtSetParent(parentNode);
            parentNodeChildCount++;
        } else // node with a new fieldname, wrapped with a marker node
        if (valuesForField.isThresholdExceeded()) {
            // create a set of nodes wrapping each pattern
            List<String> patterns = new ArrayList<>(fieldsToValues.getPatterns() == null ? new ArrayList<>() : fieldsToValues.getPatterns());
            if (patterns.isEmpty()) {
                JexlNode child = new ExceededValueThresholdMarkerJexlNode(buildUntypedNode(orgNode, field));
                parentNode.jjtAddChild(child, parentNodeChildCount);
                child.jjtSetParent(parentNode);
                parentNodeChildCount++;
            } else if (patterns.size() == 1) {
                JexlNode child = new ExceededValueThresholdMarkerJexlNode(buildUntypedNode(orgNode, field, patterns.get(0)));
                parentNode.jjtAddChild(child, parentNodeChildCount);
                child.jjtSetParent(parentNode);
                parentNodeChildCount++;
            } else {
                int childNodeChildCount = 0;
                JexlNode childNode = (containerType.equals(ContainerType.OR_NODE) ? new ASTOrNode(ParserTreeConstants.JJTORNODE) : new ASTAndNode(ParserTreeConstants.JJTANDNODE));
                JexlNodes.ensureCapacity(childNode, patterns.size());
                for (String pattern : patterns) {
                    JexlNode child = new ExceededValueThresholdMarkerJexlNode(buildUntypedNode(orgNode, field, pattern));
                    childNode.jjtAddChild(child, childNodeChildCount);
                    child.jjtSetParent(childNode);
                    childNodeChildCount++;
                }
                if (0 < childNodeChildCount) {
                    JexlNode wrappedChildNode = wrap(childNode);
                    childNode.jjtSetParent(wrappedChildNode);
                    parentNode.jjtAddChild(wrappedChildNode, parentNodeChildCount);
                    childNode.jjtSetParent(childNode);
                    parentNodeChildCount++;
                }
            }
        } else // Don't create an OR if we have only one value, directly attach it
        if (1 == valuesForField.size()) {
            JexlNode child = buildUntypedNode(node, field, valuesForField.iterator().next());
            parentNode.jjtAddChild(child, parentNodeChildCount);
            child.jjtSetParent(parentNode);
            parentNodeChildCount++;
        } else {
            int childNodeChildCount = 0;
            JexlNode childNode = (containerType.equals(ContainerType.OR_NODE) ? new ASTOrNode(ParserTreeConstants.JJTORNODE) : new ASTAndNode(ParserTreeConstants.JJTANDNODE));
            JexlNodes.ensureCapacity(childNode, valuesForField.size());
            for (String value : valuesForField) {
                JexlNode child = buildUntypedNode(node, field, value);
                childNode.jjtAddChild(child, childNodeChildCount);
                child.jjtSetParent(childNode);
                childNodeChildCount++;
            }
            if (0 < childNodeChildCount) {
                JexlNode wrappedChildNode = wrap(childNode);
                childNode.jjtSetParent(wrappedChildNode);
                parentNode.jjtAddChild(wrappedChildNode, parentNodeChildCount);
                childNode.jjtSetParent(childNode);
                parentNodeChildCount++;
            }
        }
    }
    switch(parentNodeChildCount) {
        case 0:
            // in this case we had no matches for the range, so this expression gets replaced with a FALSE node.
            return new ASTFalseNode(ParserTreeConstants.JJTFALSENODE);
        case 1:
            JexlNode child = parentNode.jjtGetChild(0);
            JexlNodes.promote(parentNode, child);
            return child;
        default:
            JexlNode wrappedParentNode = wrap(parentNode);
            parentNode.jjtSetParent(wrappedParentNode);
            return wrappedParentNode;
    }
}