Examples with Composite datawave.query.composite.Composite used on opensource projects

Example 1 with Composite

use of datawave.query.composite.Composite in project datawave by NationalSecurityAgency.

the class ExpandCompositeTerms method createCompositeNode.

/**
 * Attempts to form a jexl node from the composite
 *
 * @param composite
 *            A list of composites from which jexl nodes should be created
 * @return A list of jexl nodes created from the given composite
 */
private JexlNode createCompositeNode(Composite composite) {
    List<Class<? extends JexlNode>> nodeClasses = new ArrayList<>();
    List<String> appendedExpressions = new ArrayList<>();
    boolean includeOldData = false;
    if (config.getCompositeTransitionDates().containsKey(composite.getCompositeName())) {
        Date transitionDate = config.getCompositeTransitionDates().get(composite.getCompositeName());
        if (config.getBeginDate().compareTo(transitionDate) < 0)
            includeOldData = true;
    }
    composite.getNodesAndExpressions(nodeClasses, appendedExpressions, config.getFieldToDiscreteIndexTypes(), includeOldData);
    // if this is true, then it indicates that we are dealing with a query containing an overloaded composite
    // field which only contained the first component term. This means that we are running a query against
    // the base composite term, and thus need to expand our ranges to fully include both the composite and
    // non-composite events in our range.
    boolean expandRangeForBaseTerm = CompositeIngest.isOverloadedCompositeField(config.getCompositeToFieldMap(), composite.getCompositeName()) && composite.getJexlNodeList().size() == 1;
    DiscreteIndexType<?> baseTermDiscreteIndexType = config.getFieldToDiscreteIndexTypes().get(composite.getFieldNameList().get(0));
    List<JexlNode> finalNodes = new ArrayList<>();
    for (int i = 0; i < nodeClasses.size(); i++) {
        Class<? extends JexlNode> nodeClass = nodeClasses.get(i);
        String appendedExpression = appendedExpressions.get(i);
        JexlNode newNode = null;
        if (nodeClass.equals(ASTGTNode.class)) {
            if (expandRangeForBaseTerm)
                newNode = JexlNodeFactory.buildNode((ASTGENode) null, composite.getCompositeName(), CompositeUtils.getInclusiveLowerBound(appendedExpression, baseTermDiscreteIndexType));
            else
                newNode = JexlNodeFactory.buildNode((ASTGTNode) null, composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTGENode.class)) {
            newNode = JexlNodeFactory.buildNode((ASTGENode) null, composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTLTNode.class)) {
            newNode = JexlNodeFactory.buildNode((ASTLTNode) null, composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTLENode.class)) {
            if (expandRangeForBaseTerm)
                newNode = JexlNodeFactory.buildNode((ASTLTNode) null, composite.getCompositeName(), CompositeUtils.getExclusiveUpperBound(appendedExpression, baseTermDiscreteIndexType));
            else
                newNode = JexlNodeFactory.buildNode((ASTLENode) null, composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTERNode.class)) {
            newNode = JexlNodeFactory.buildERNode(composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTNENode.class)) {
            newNode = JexlNodeFactory.buildNode((ASTNENode) null, composite.getCompositeName(), appendedExpression);
        } else if (nodeClass.equals(ASTEQNode.class)) {
            // if this is for an overloaded composite field, which only includes the base term, convert to range
            if (expandRangeForBaseTerm) {
                JexlNode lowerBound = JexlNodeFactory.buildNode((ASTGENode) null, composite.getCompositeName(), appendedExpression);
                JexlNode upperBound = JexlNodeFactory.buildNode((ASTLTNode) null, composite.getCompositeName(), CompositeUtils.getExclusiveUpperBound(appendedExpression, baseTermDiscreteIndexType));
                newNode = createUnwrappedAndNode(Arrays.asList(lowerBound, upperBound));
            } else {
                newNode = JexlNodeFactory.buildEQNode(composite.getCompositeName(), appendedExpression);
            }
        } else {
            log.error("Invalid or unknown node type for composite map.");
        }
        finalNodes.add(newNode);
    }
    JexlNode finalNode;
    if (finalNodes.size() > 1) {
        finalNode = createUnwrappedAndNode(finalNodes);
        if (composite.getJexlNodeList().size() > 1) {
            JexlNode delayedNode = ASTEvaluationOnly.create(createUnwrappedAndNode(composite.getJexlNodeList().stream().map(node -> JexlNodeFactory.wrap(copy(node))).collect(Collectors.toList())));
            finalNode = createUnwrappedAndNode(Arrays.asList(JexlNodeFactory.wrap(finalNode), delayedNode));
        }
    } else {
        finalNode = finalNodes.get(0);
        if (composite.getJexlNodeList().size() > 1 && !(finalNode instanceof ASTEQNode)) {
            JexlNode delayedNode = ASTEvaluationOnly.create(createUnwrappedAndNode(composite.getJexlNodeList().stream().map(node -> JexlNodeFactory.wrap(copy(node))).collect(Collectors.toList())));
            finalNode = createUnwrappedAndNode(Arrays.asList(finalNode, delayedNode));
        }
    }
    if (!CompositeIngest.isOverloadedCompositeField(config.getCompositeToFieldMap(), composite.getCompositeName())) {
        config.getIndexedFields().add(composite.getCompositeName());
        config.getQueryFieldsDatatypes().put(composite.getCompositeName(), new NoOpType());
    }
    // save a mapping of generated composites to their component parts for later processing
    jexlNodeToCompMap.put(finalNode, composite);
    return finalNode;
}

Example 2 with Composite

use of datawave.query.composite.Composite in project datawave by NationalSecurityAgency.

the class ExpandCompositeTerms method visitLeafNode.

/**
 * The default leaf node visitor, which uses the anded nodes to determine whether a composite can be formed with this leaf node.
 *
 * @param node
 *            A leaf node from the original script
 * @param eData
 *            ExpandData, containing ancestor anded nodes, used anded nodes, and a flag indicating whether composites were found
 * @return Returns a composite node if one can be made, otherwise returns the original node
 */
private JexlNode visitLeafNode(JexlNode node, ExpandData eData) {
    LiteralRange range = JexlASTHelper.findRange().getRange(node);
    String fieldName = (range != null) ? range.getFieldName() : JexlASTHelper.getIdentifier(node);
    Multimap<String, JexlNode> leafNodes = LinkedHashMultimap.create();
    leafNodes.put(fieldName, node);
    List<Composite> foundComposites = findComposites(leafNodes, eData.andedNodes, eData.usedAndedNodes);
    JexlNode resultNode = node;
    // if composites were found, create JexlNodes from them
    if (!foundComposites.isEmpty()) {
        List<JexlNode> compositeNodes = createCompositeNodes(foundComposites);
        if (!compositeNodes.isEmpty()) {
            eData.foundComposite = true;
            resultNode = createUnwrappedAndNode(compositeNodes);
        }
    }
    return resultNode;
}

Example 3 with Composite

use of datawave.query.composite.Composite in project datawave by NationalSecurityAgency.

the class ExpandCompositeTerms method findComposites.

/**
 * Returns a list of composites that can be generated from the given leaf nodes and anded nodes. The used leaf nodes and anded nodes will be returned via
 * their respective parameters.
 *
 * @param compositeToFieldMapSet
 *            A set of composite fields, mapped to their component fields which can be created with the given leaf and anded nodes
 * @param leafNodes
 *            A multimap of leaf child nodes, keyed by field name, from the parent node
 * @param andedNodes
 *            A multimap of anded nodes, keyed by field name, passed down from our ancestors
 * @param usedLeafNodes
 *            A multimap of used leaf child nodes, keyed by field name, used to create the returned composites
 * @param usedAndedNodes
 *            A multimap of used anded nodes, keyed by field name, used to create the returned composites
 * @return A list of composites which can be created from the given leaf and anded nodes
 */
private List<Composite> findComposites(List<Entry<String, Collection<String>>> compositeToFieldMapSet, Multimap<String, JexlNode> leafNodes, Multimap<String, JexlNode> andedNodes, Multimap<String, JexlNode> usedLeafNodes, Multimap<String, JexlNode> usedAndedNodes) {
    List<Composite> composites = new ArrayList<>();
    // once a leaf node is used to create a composite, take it out of the rotation
    Multimap<String, JexlNode> remainingLeafNodes = LinkedHashMultimap.create();
    remainingLeafNodes.putAll(leafNodes);
    // once an anded node is used to create a composite, take it out of the rotation
    Multimap<String, JexlNode> remainingAndedNodes = LinkedHashMultimap.create();
    remainingAndedNodes.putAll(andedNodes);
    // look at each potential composite name to see if its fields are all available as keys in the childNodeMap
    for (Map.Entry<String, Collection<String>> compositeToFieldMap : compositeToFieldMapSet) {
        String compositeField = compositeToFieldMap.getKey();
        List<String> componentFields = new ArrayList<>(compositeToFieldMap.getValue());
        // is this a query against a composite field with old data whose date range predates the transition date?
        if (CompositeIngest.isOverloadedCompositeField(config.getCompositeToFieldMap(), compositeField) && config.getCompositeTransitionDates().containsKey(compositeField)) {
            Date transitionDate = config.getCompositeTransitionDates().get(compositeField);
            if (config.getEndDate().compareTo(transitionDate) < 0)
                continue;
        }
        // @formatter:off
        boolean leafNodeFieldPresent = componentFields.stream().anyMatch(componentField -> remainingLeafNodes.keySet().contains(componentField));
        // only build this composite if one of the components is a leaf node
        if (!leafNodeFieldPresent)
            continue;
        // @formatter:off
        boolean allRequiredFieldsPresent = componentFields.stream().allMatch(componentField -> remainingLeafNodes.keySet().contains(componentField) || remainingAndedNodes.keySet().contains(componentField));
        // only build this composite if we have all of the required component fields
        if (!allRequiredFieldsPresent)
            continue;
        // we have what we need to make a composite
        List<Composite> tempComposites = new ArrayList<>();
        Composite baseComp = new CompositeTerm(compositeField, config.getCompositeFieldSeparators().get(compositeField));
        tempComposites.add(baseComp);
        // keep track of the used nodes
        Multimap<String, JexlNode> tempUsedLeafNodes = LinkedHashMultimap.create();
        Multimap<String, JexlNode> tempUsedAndedNodes = LinkedHashMultimap.create();
        // composite, creating additional nodes when necessary
        for (int i = 0; i < componentFields.size(); i++) {
            String componentField = componentFields.get(i);
            Collection<JexlNode> nodes = Lists.newArrayList();
            // add any required leaf nodes
            for (JexlNode node : remainingLeafNodes.get(componentField)) {
                JexlNode trimmedNode = getLeafNode(node);
                if (trimmedNode != null && isNodeValid(trimmedNode, i, componentFields.size(), isFixedLengthField(componentField))) {
                    nodes.add(trimmedNode);
                    tempUsedLeafNodes.put(componentField, node);
                }
            }
            // add any required anded nodes
            for (JexlNode node : remainingAndedNodes.get(componentField)) {
                JexlNode trimmedNode = getLeafNode(node);
                if (trimmedNode != null && isNodeValid(trimmedNode, i, componentFields.size(), isFixedLengthField(componentField))) {
                    nodes.add(trimmedNode);
                    tempUsedAndedNodes.put(componentField, node);
                }
            }
            // if at any point we run out of eligible nodes, then we have failed to build the composite, and need to stop
            if (nodes.isEmpty()) {
                tempComposites.clear();
                break;
            }
            tempComposites = updateComposites(tempComposites, nodes);
            // then we failed to update the composites
            if (tempComposites.isEmpty())
                break;
        }
        if (!tempComposites.isEmpty()) {
            // save the found composites
            composites.addAll(tempComposites);
            // keep track of the used nodes
            if (usedLeafNodes != null)
                usedLeafNodes.putAll(tempUsedLeafNodes);
            if (usedAndedNodes != null)
                usedAndedNodes.putAll(tempUsedAndedNodes);
            // take the used nodes out of the rotation
            for (Entry<String, JexlNode> usedNode : tempUsedLeafNodes.entries()) remainingLeafNodes.remove(usedNode.getKey(), usedNode.getValue());
            for (Entry<String, JexlNode> usedNode : tempUsedAndedNodes.entries()) remainingAndedNodes.remove(usedNode.getKey(), usedNode.getValue());
        }
    }
    return composites;
}

Example 4 with Composite

use of datawave.query.composite.Composite in project datawave by NationalSecurityAgency.

the class ExpandCompositeTerms method updateComposites.

/**
 * This method will add the given nodes to the passed in composites. If an invalid composite is produced as a result of adding in one of the nodes, we will
 * abort creation of this composite.
 *
 * @param composites
 *            Composite objects being tracked for a given key
 * @param nodes
 *            Collection of nodes matching a single field
 * @return A list of updated composites
 */
private List<Composite> updateComposites(List<Composite> composites, Collection<JexlNode> nodes) {
    List<Composite> updatedComposites = new ArrayList<>();
    // add each of the nodes to each of the composites
    for (JexlNode node : nodes) {
        for (Composite composite : composites) {
            Composite updatedComposite;
            // if this is a node which would be invalid for a normal composite, create a composite range
            if (!CompositeTerm.VALID_LEAF_NODE_CLASSES.contains(node.getClass()) && CompositeRange.VALID_LEAF_NODE_CLASSES.contains(node.getClass())) {
                updatedComposite = new CompositeRange(composite);
            } else // if there is more than 1 node to process, clone the composite
            if (nodes.size() > 1) {
                updatedComposite = composite.clone();
            } else // if there is only 1 node to process, just update the composite
            {
                updatedComposite = composite;
            }
            // update the composite
            updatedComposite.addComponent(node);
            // is it valid? if not, we can't build the composite
            if (!updatedComposite.isValid()) {
                updatedComposites.clear();
                break;
            }
            // save the updated composite
            updatedComposites.add(updatedComposite);
        }
        // composites, something went wrong
        if (updatedComposites.isEmpty())
            break;
    }
    return updatedComposites;
}

Example 5 with Composite

use of datawave.query.composite.Composite in project datawave by NationalSecurityAgency.

the class ExpandCompositeTerms method processUnusedLeafNodes.

/**
 * Attempts to create composites using the remaining leaf and anded nodes from our ancestors. Each of the composites created must contain at least one of
 * the leaf nodes in order to be valid. The used leaf nodes are passed back via the usedLeafNodes param. The used anded nodes are passed back via the
 * parentData.
 *
 * @param parentData
 *            Contains the ancestor anded nodes, anded nodes used to create the returned composites, and a flag indicating whether composites were found
 * @param leafNodes
 *            A multimap of leaf child nodes, keyed by field name, from the parent node
 * @param usedLeafNodes
 *            A multimap of used leaf child nodes, keyed by field name, used to create the returned composites
 * @return A list of modified and unmodified leaf child nodes, from the parent node
 */
private List<JexlNode> processUnusedLeafNodes(ExpandData parentData, Multimap<String, JexlNode> leafNodes, Multimap<String, JexlNode> usedLeafNodes) {
    // use the remaining leaf and anded nodes to generate composites
    // note: the used leaf and anded nodes are removed in 'processNonLeafNodes'
    List<Composite> foundComposites = findComposites(leafNodes, parentData.andedNodes, usedLeafNodes, parentData.usedAndedNodes);
    List<JexlNode> compositeLeafNodes = new ArrayList<>();
    // if we found some composites
    if (!foundComposites.isEmpty()) {
        List<JexlNode> compositeNodes = createCompositeNodes(foundComposites);
        // add the composite nodes to our list of processed nodes
        if (!compositeNodes.isEmpty()) {
            parentData.foundComposite = true;
            compositeLeafNodes.add(createUnwrappedAndNode(compositeNodes));
        }
    }
    leafNodes.values().removeAll(usedLeafNodes.values());
    List<JexlNode> unmodifiedLeafNodes = new ArrayList<>();
    List<JexlNode> modifiedLeafNodes = new ArrayList<>();
    // overloaded composite whose bounds were adjusted
    for (JexlNode remainingLeafNode : leafNodes.values()) {
        ExpandData eData = new ExpandData();
        JexlNode processedNode = (JexlNode) remainingLeafNode.jjtAccept(this, eData);
        if (eData.foundComposite) {
            parentData.foundComposite = true;
            modifiedLeafNodes.add(processedNode);
        } else {
            unmodifiedLeafNodes.add(remainingLeafNode);
        }
    }
    List<JexlNode> processedLeafNodes = new ArrayList<>();
    processedLeafNodes.addAll(unmodifiedLeafNodes);
    processedLeafNodes.addAll(modifiedLeafNodes);
    processedLeafNodes.addAll(compositeLeafNodes);
    return processedLeafNodes;
}