Search in sources :

Example 1 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class GroupingDocumentTransformer method createGroupFieldsList.

public void createGroupFieldsList(Collection<String> groupFieldsSet) {
    this.groupFieldsList = Lists.newArrayList(groupFieldsSet);
    QueryModel model = ((ShardQueryLogic) logic).getQueryModel();
    for (String groupField : groupFieldsSet) {
        String f = model.getReverseAliasForField(groupField);
        if (f != null && !f.isEmpty()) {
            this.groupFieldsList.add(f);
        }
    }
}
Also used : ShardQueryLogic(datawave.query.tables.ShardQueryLogic) QueryModel(datawave.query.model.QueryModel)

Example 2 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class QueryModelLoader method store.

/*
     * (non-Javadoc)
     * 
     * @see datawave.util.cache.AccumuloLoader#store(java.lang.Object, org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value)
     */
@Override
protected boolean store(Entry<String, String> storeKey, Key key, Value value) {
    if (log.isDebugEnabled())
        log.debug("== Requesting to store (" + storeKey.getKey() + "," + storeKey.getValue() + ") " + key);
    // will hold index only fields
    Set<String> indexOnlyFields = new HashSet<>();
    // We need the entire Model so we can do both directions.
    Entry<QueryModel, Set<String>> modelEntry = entryCache.get(storeKey);
    QueryModel queryModel = null;
    if (null == modelEntry)
        queryModel = new QueryModel();
    else
        queryModel = modelEntry.getKey();
    if (null != key.getColumnQualifier()) {
        String original = key.getRow().toString();
        Text cq = key.getColumnQualifier();
        String[] parts = StringUtils.split(cq.toString(), "\0");
        if (parts.length > 1 && null != parts[0] && !parts[0].isEmpty()) {
            String replacement = parts[0];
            for (String part : parts) {
                if ("forward".equalsIgnoreCase(part)) {
                    // when the replacement does not exist in the database
                    if (allFields == null || allFields.contains(replacement)) {
                        queryModel.addTermToModel(original, replacement);
                    } else if (log.isTraceEnabled()) {
                        log.trace("Ignoring forward mapping of " + replacement + " for " + original + " because the metadata table has no reference to it");
                    }
                } else if ("reverse".equalsIgnoreCase(part)) {
                    queryModel.addTermToReverseModel(original, replacement);
                } else if ("index_only".equalsIgnoreCase(part)) {
                    indexOnlyFields.add(replacement);
                }
            }
        }
    }
    entryCache.put(storeKey, Maps.immutableEntry(queryModel, indexOnlyFields));
    return true;
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) Text(org.apache.hadoop.io.Text) QueryModel(datawave.query.model.QueryModel) HashSet(java.util.HashSet)

Example 3 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class QueryModelVisitor method expandBinaryNodeFromModel.

/**
 * Applies the forward mapping from the QueryModel to a node, expanding the node into an Or if needed.
 *
 * @param node
 * @param data
 * @return
 */
protected JexlNode expandBinaryNodeFromModel(JexlNode node, Object data) {
    String field = JexlASTHelper.getIdentifier(node);
    if (isFieldExcluded(field)) {
        return node;
    }
    // Count the immediate children:
    int childCount = node.jjtGetNumChildren();
    if (childCount != 2) {
        QueryException qe = new QueryException(DatawaveErrorCode.BINARY_NODE_TOO_MANY_CHILDREN, MessageFormat.format("Node: {0}", PrintingVisitor.formattedQueryString(node)));
        throw new DatawaveFatalQueryException(qe);
    }
    // Find identifiers
    List<ASTIdentifier> allidentifiers = JexlASTHelper.getIdentifiers(node);
    // If we don't have any identifiers, we have nothing to expand
    if (allidentifiers.isEmpty()) {
        return node;
    }
    JexlNode leftNode = node.jjtGetChild(0);
    JexlNode rightNode = node.jjtGetChild(1);
    if (log.isTraceEnabled()) {
        log.trace("leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // this will expand identifiers that have a method connected to them
    boolean leftState = JexlASTHelper.HasMethodVisitor.hasMethod(leftNode);
    if (leftState) {
        // there is a method under leftNode
        leftNode = (JexlNode) leftNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    boolean rightState = JexlASTHelper.HasMethodVisitor.hasMethod(rightNode);
    if (rightState) {
        // there is a method under rightNode
        rightNode = (JexlNode) rightNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    // expand any identifiers inside of methods/functions in the left and right nodes
    leftNode = (JexlNode) leftNode.jjtAccept(this, null);
    rightNode = (JexlNode) rightNode.jjtAccept(this, null);
    if (log.isTraceEnabled()) {
        log.trace("after expansion, leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("after expansion, leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("after expansion, rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("after expansion, rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // if state == true on either side, then there is a method on one side and we are done applying the model
    if (leftState || rightState) {
        JexlNode toReturn = JexlNodeFactory.buildUntypedBinaryNode(node, leftNode, rightNode);
        if (log.isTraceEnabled()) {
            log.trace("done early. returning:" + JexlStringBuildingVisitor.buildQuery(toReturn));
        }
        return toReturn;
    }
    JexlNode leftSeed, rightSeed;
    Set<JexlNode> left = Sets.newHashSet(), right = Sets.newHashSet();
    boolean isNullEquality = false;
    if (node instanceof ASTEQNode && (leftNode instanceof ASTNullLiteral || rightNode instanceof ASTNullLiteral)) {
        isNullEquality = true;
    }
    // the query has been previously groomed so that identifiers are on the left and literals are on the right
    // an identifier with a method attached will have already been substituted above (and will return null for the IdentifierOpLiteral)
    // The normal case of `IDENTIFIER op 'literal'`
    JexlASTHelper.IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op != null) {
        // One identifier
        leftSeed = op.getIdentifier();
        rightSeed = op.getLiteral();
        if (rightSeed instanceof ASTNullLiteral && node instanceof ASTEQNode) {
            isNullEquality = true;
        }
    } else {
        // We know from above that childCount == 2. We may have a reference on both sides of the expression
        leftSeed = node.jjtGetChild(0);
        rightSeed = node.jjtGetChild(1);
    }
    if (leftSeed instanceof ASTReference) {
        // String fieldName = JexlASTHelper.getIdentifier((JexlNode)leftSeed);
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(leftSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(leftSeed) + " from " + JexlStringBuildingVisitor.buildQuery(leftNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                left.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (leftSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) leftSeed))) {
            left.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        left.add(leftSeed);
    }
    if (rightSeed instanceof ASTReference) {
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(rightSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(rightSeed) + " from " + JexlStringBuildingVisitor.buildQuery(rightNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                right.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (rightSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) rightSeed))) {
            right.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        right.add(rightSeed);
    }
    boolean requiresAnd = isNullEquality || node instanceof ASTNENode;
    @SuppressWarnings("unchecked") Set<List<JexlNode>> // retrieve the cartesian product
    product = Sets.cartesianProduct(left, right);
    /**
     * use the product transformer to shallow copy the jexl nodes. We've created new nodes that will be embedded within an ast reference. As a result, we
     * need to ensure that if we create a logical structure ( such as an or ) -- each literal references a unique identifier from the right. Otherwise,
     * subsequent visitors will reference incorrection sub trees, and potentially negate the activity of the query model visitor
     */
    Set<List<JexlNode>> newSet = product.stream().map(list -> list.stream().map(RebuildingVisitor::copy).collect(Collectors.toList())).collect(Collectors.toSet());
    if (product.size() > 1) {
        JexlNode expanded;
        if (requiresAnd) {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.AND_NODE, node, newSet);
        } else {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.OR_NODE, node, newSet);
        }
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    } else if (1 == product.size()) {
        List<JexlNode> pair = product.iterator().next();
        JexlNode expanded = JexlNodeFactory.buildUntypedBinaryNode(node, pair.get(0), pair.get(1));
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    }
    // If we couldn't map anything, return a copy
    if (log.isTraceEnabled())
        log.trace("just returning the original:" + PrintingVisitor.formattedQueryString(node));
    return node;
}
Also used : ContainerType(datawave.query.jexl.JexlNodeFactory.ContainerType) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) ASTLTNode(org.apache.commons.jexl2.parser.ASTLTNode) JexlASTHelper(datawave.query.jexl.JexlASTHelper) ASTLENode(org.apache.commons.jexl2.parser.ASTLENode) ASTMethodNode(org.apache.commons.jexl2.parser.ASTMethodNode) QueryModel(datawave.query.model.QueryModel) JexlNode(org.apache.commons.jexl2.parser.JexlNode) JexlNodeFactory(datawave.query.jexl.JexlNodeFactory) ASTSizeMethod(org.apache.commons.jexl2.parser.ASTSizeMethod) MessageFormat(java.text.MessageFormat) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ASTERNode(org.apache.commons.jexl2.parser.ASTERNode) Logger(org.apache.log4j.Logger) Lists(com.google.common.collect.Lists) ASTNRNode(org.apache.commons.jexl2.parser.ASTNRNode) LiteralRange(datawave.query.jexl.LiteralRange) ASTGTNode(org.apache.commons.jexl2.parser.ASTGTNode) ASTAndNode(org.apache.commons.jexl2.parser.ASTAndNode) ThreadConfigurableLogger(datawave.webservice.common.logging.ThreadConfigurableLogger) ParserTreeConstants(org.apache.commons.jexl2.parser.ParserTreeConstants) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) ASTNullLiteral(org.apache.commons.jexl2.parser.ASTNullLiteral) ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) JexlNodes(org.apache.commons.jexl2.parser.JexlNodes) BoundedRange(datawave.query.jexl.nodes.BoundedRange) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) DatawaveErrorCode(datawave.webservice.query.exception.DatawaveErrorCode) Node(org.apache.commons.jexl2.parser.Node) ASTNENode(org.apache.commons.jexl2.parser.ASTNENode) ASTGENode(org.apache.commons.jexl2.parser.ASTGENode) List(java.util.List) QueryException(datawave.webservice.query.exception.QueryException) ASTOrNode(org.apache.commons.jexl2.parser.ASTOrNode) ASTFunctionNode(org.apache.commons.jexl2.parser.ASTFunctionNode) Collections(java.util.Collections) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) ASTReference(org.apache.commons.jexl2.parser.ASTReference) ASTNENode(org.apache.commons.jexl2.parser.ASTNENode) ASTIdentifier(org.apache.commons.jexl2.parser.ASTIdentifier) ASTNullLiteral(org.apache.commons.jexl2.parser.ASTNullLiteral) ASTReference(org.apache.commons.jexl2.parser.ASTReference) JexlASTHelper(datawave.query.jexl.JexlASTHelper) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) QueryException(datawave.webservice.query.exception.QueryException) ASTEQNode(org.apache.commons.jexl2.parser.ASTEQNode) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) JexlNode(org.apache.commons.jexl2.parser.JexlNode) ArrayList(java.util.ArrayList) List(java.util.List)

Example 4 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class LoadModelFromXml method loadModelFromXml.

public static QueryModel loadModelFromXml(InputStream stream) throws Exception {
    SAXParserFactory spf = SAXParserFactory.newInstance();
    spf.setFeature("http://xml.org/sax/features/external-general-entities", false);
    spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
    spf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
    spf.setNamespaceAware(true);
    Source xmlSource = new SAXSource(spf.newSAXParser().getXMLReader(), new InputSource(stream));
    JAXBContext ctx = JAXBContext.newInstance(Model.class);
    Unmarshaller um = ctx.createUnmarshaller();
    Model xmlModel = (Model) um.unmarshal(xmlSource);
    if (log.isDebugEnabled()) {
        log.debug(xmlModel.getName());
        for (FieldMapping fieldMapping : xmlModel.getFields()) {
            log.debug(fieldMapping.toString());
        }
    }
    QueryModel model = new QueryModel();
    for (FieldMapping mapping : xmlModel.getFields()) {
        switch(mapping.getDirection()) {
            case FORWARD:
                model.addTermToModel(mapping.getModelFieldName(), mapping.getFieldName());
                break;
            case REVERSE:
                model.addTermToReverseModel(mapping.getFieldName(), mapping.getModelFieldName());
                break;
            default:
                log.error("Unknown direction: " + mapping.getDirection());
        }
    }
    if (model.getForwardQueryMapping().isEmpty() && model.getReverseQueryMapping().isEmpty()) {
        throw new IllegalArgumentException("The resulting, loaded query model was empty.");
    }
    return model;
}
Also used : InputSource(org.xml.sax.InputSource) SAXSource(javax.xml.transform.sax.SAXSource) FieldMapping(datawave.webservice.model.FieldMapping) Model(datawave.webservice.model.Model) QueryModel(datawave.query.model.QueryModel) JAXBContext(javax.xml.bind.JAXBContext) Unmarshaller(javax.xml.bind.Unmarshaller) QueryModel(datawave.query.model.QueryModel) InputSource(org.xml.sax.InputSource) SAXSource(javax.xml.transform.sax.SAXSource) Source(javax.xml.transform.Source) SAXParserFactory(javax.xml.parsers.SAXParserFactory)

Example 5 with QueryModel

use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.

the class MetadataHelperQueryModelProvider method getQueryModel.

@Override
public QueryModel getQueryModel() {
    QueryModel queryModel = null;
    if (config.getQueryModel() != null) {
        log.debug("Using a cached query model");
        queryModel = config.getQueryModel();
    } else if (null != config.getModelName() && null != config.getModelTableName()) {
        log.debug("Generating a query model");
        try {
            queryModel = metadataHelper.getQueryModel(config.getModelTableName(), config.getModelName(), config.getUnevaluatedFields(), config.getDatatypeFilter());
            config.setQueryModel(queryModel);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.QUERY_MODEL_FETCH_ERROR, e);
            log.error(qe);
            throw new DatawaveFatalQueryException(qe);
        }
        if (log.isTraceEnabled()) {
            log.trace("forward queryModel: " + queryModel.getForwardQueryMapping());
            log.trace("reverse queryModel: " + queryModel.getReverseQueryMapping());
        }
    }
    return queryModel;
}
Also used : TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) QueryException(datawave.webservice.query.exception.QueryException) DatawaveFatalQueryException(datawave.query.exceptions.DatawaveFatalQueryException) QueryModel(datawave.query.model.QueryModel)

Aggregations

QueryModel (datawave.query.model.QueryModel)13 DatawaveFatalQueryException (datawave.query.exceptions.DatawaveFatalQueryException)3 HashSet (java.util.HashSet)3 Test (org.junit.Test)3 FieldMapping (datawave.webservice.model.FieldMapping)2 Model (datawave.webservice.model.Model)2 QueryImpl (datawave.webservice.query.QueryImpl)2 QueryException (datawave.webservice.query.exception.QueryException)2 Set (java.util.Set)2 JAXBContext (javax.xml.bind.JAXBContext)2 Unmarshaller (javax.xml.bind.Unmarshaller)2 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 DateType (datawave.data.type.DateType)1 DiscreteIndexType (datawave.data.type.DiscreteIndexType)1 GeometryType (datawave.data.type.GeometryType)1 NoOpType (datawave.data.type.NoOpType)1 StringType (datawave.data.type.StringType)1 Type (datawave.data.type.Type)1 UniqueFields (datawave.query.attributes.UniqueFields)1