use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class GroupingDocumentTransformer method createGroupFieldsList.
public void createGroupFieldsList(Collection<String> groupFieldsSet) {
this.groupFieldsList = Lists.newArrayList(groupFieldsSet);
QueryModel model = ((ShardQueryLogic) logic).getQueryModel();
for (String groupField : groupFieldsSet) {
String f = model.getReverseAliasForField(groupField);
if (f != null && !f.isEmpty()) {
this.groupFieldsList.add(f);
}
}
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class QueryModelLoader method store.
/*
* (non-Javadoc)
*
* @see datawave.util.cache.AccumuloLoader#store(java.lang.Object, org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value)
*/
@Override
protected boolean store(Entry<String, String> storeKey, Key key, Value value) {
if (log.isDebugEnabled())
log.debug("== Requesting to store (" + storeKey.getKey() + "," + storeKey.getValue() + ") " + key);
// will hold index only fields
Set<String> indexOnlyFields = new HashSet<>();
// We need the entire Model so we can do both directions.
Entry<QueryModel, Set<String>> modelEntry = entryCache.get(storeKey);
QueryModel queryModel = null;
if (null == modelEntry)
queryModel = new QueryModel();
else
queryModel = modelEntry.getKey();
if (null != key.getColumnQualifier()) {
String original = key.getRow().toString();
Text cq = key.getColumnQualifier();
String[] parts = StringUtils.split(cq.toString(), "\0");
if (parts.length > 1 && null != parts[0] && !parts[0].isEmpty()) {
String replacement = parts[0];
for (String part : parts) {
if ("forward".equalsIgnoreCase(part)) {
// when the replacement does not exist in the database
if (allFields == null || allFields.contains(replacement)) {
queryModel.addTermToModel(original, replacement);
} else if (log.isTraceEnabled()) {
log.trace("Ignoring forward mapping of " + replacement + " for " + original + " because the metadata table has no reference to it");
}
} else if ("reverse".equalsIgnoreCase(part)) {
queryModel.addTermToReverseModel(original, replacement);
} else if ("index_only".equalsIgnoreCase(part)) {
indexOnlyFields.add(replacement);
}
}
}
}
entryCache.put(storeKey, Maps.immutableEntry(queryModel, indexOnlyFields));
return true;
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class QueryModelVisitor method expandBinaryNodeFromModel.
/**
* Applies the forward mapping from the QueryModel to a node, expanding the node into an Or if needed.
*
* @param node
* @param data
* @return
*/
protected JexlNode expandBinaryNodeFromModel(JexlNode node, Object data) {
String field = JexlASTHelper.getIdentifier(node);
if (isFieldExcluded(field)) {
return node;
}
// Count the immediate children:
int childCount = node.jjtGetNumChildren();
if (childCount != 2) {
QueryException qe = new QueryException(DatawaveErrorCode.BINARY_NODE_TOO_MANY_CHILDREN, MessageFormat.format("Node: {0}", PrintingVisitor.formattedQueryString(node)));
throw new DatawaveFatalQueryException(qe);
}
// Find identifiers
List<ASTIdentifier> allidentifiers = JexlASTHelper.getIdentifiers(node);
// If we don't have any identifiers, we have nothing to expand
if (allidentifiers.isEmpty()) {
return node;
}
JexlNode leftNode = node.jjtGetChild(0);
JexlNode rightNode = node.jjtGetChild(1);
if (log.isTraceEnabled()) {
log.trace("leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
log.trace("leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
log.trace("rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
log.trace("rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
}
// this will expand identifiers that have a method connected to them
boolean leftState = JexlASTHelper.HasMethodVisitor.hasMethod(leftNode);
if (leftState) {
// there is a method under leftNode
leftNode = (JexlNode) leftNode.jjtAccept(this.simpleQueryModelVisitor, null);
}
boolean rightState = JexlASTHelper.HasMethodVisitor.hasMethod(rightNode);
if (rightState) {
// there is a method under rightNode
rightNode = (JexlNode) rightNode.jjtAccept(this.simpleQueryModelVisitor, null);
}
// expand any identifiers inside of methods/functions in the left and right nodes
leftNode = (JexlNode) leftNode.jjtAccept(this, null);
rightNode = (JexlNode) rightNode.jjtAccept(this, null);
if (log.isTraceEnabled()) {
log.trace("after expansion, leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
log.trace("after expansion, leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
log.trace("after expansion, rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
log.trace("after expansion, rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
}
// if state == true on either side, then there is a method on one side and we are done applying the model
if (leftState || rightState) {
JexlNode toReturn = JexlNodeFactory.buildUntypedBinaryNode(node, leftNode, rightNode);
if (log.isTraceEnabled()) {
log.trace("done early. returning:" + JexlStringBuildingVisitor.buildQuery(toReturn));
}
return toReturn;
}
JexlNode leftSeed, rightSeed;
Set<JexlNode> left = Sets.newHashSet(), right = Sets.newHashSet();
boolean isNullEquality = false;
if (node instanceof ASTEQNode && (leftNode instanceof ASTNullLiteral || rightNode instanceof ASTNullLiteral)) {
isNullEquality = true;
}
// the query has been previously groomed so that identifiers are on the left and literals are on the right
// an identifier with a method attached will have already been substituted above (and will return null for the IdentifierOpLiteral)
// The normal case of `IDENTIFIER op 'literal'`
JexlASTHelper.IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
if (op != null) {
// One identifier
leftSeed = op.getIdentifier();
rightSeed = op.getLiteral();
if (rightSeed instanceof ASTNullLiteral && node instanceof ASTEQNode) {
isNullEquality = true;
}
} else {
// We know from above that childCount == 2. We may have a reference on both sides of the expression
leftSeed = node.jjtGetChild(0);
rightSeed = node.jjtGetChild(1);
}
if (leftSeed instanceof ASTReference) {
// String fieldName = JexlASTHelper.getIdentifier((JexlNode)leftSeed);
List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(leftSeed);
if (identifiers.size() > 1) {
log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(leftSeed) + " from " + JexlStringBuildingVisitor.buildQuery(leftNode));
}
for (ASTIdentifier identifier : identifiers) {
for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
left.add(JexlNodeFactory.buildIdentifier(fieldName));
}
}
} else if (leftSeed instanceof ASTIdentifier) {
for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) leftSeed))) {
left.add(JexlNodeFactory.buildIdentifier(fieldName));
}
} else {
// Not an identifier, therefore it's probably a literal
left.add(leftSeed);
}
if (rightSeed instanceof ASTReference) {
List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(rightSeed);
if (identifiers.size() > 1) {
log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(rightSeed) + " from " + JexlStringBuildingVisitor.buildQuery(rightNode));
}
for (ASTIdentifier identifier : identifiers) {
for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
right.add(JexlNodeFactory.buildIdentifier(fieldName));
}
}
} else if (rightSeed instanceof ASTIdentifier) {
for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) rightSeed))) {
right.add(JexlNodeFactory.buildIdentifier(fieldName));
}
} else {
// Not an identifier, therefore it's probably a literal
right.add(rightSeed);
}
boolean requiresAnd = isNullEquality || node instanceof ASTNENode;
@SuppressWarnings("unchecked") Set<List<JexlNode>> // retrieve the cartesian product
product = Sets.cartesianProduct(left, right);
/**
* use the product transformer to shallow copy the jexl nodes. We've created new nodes that will be embedded within an ast reference. As a result, we
* need to ensure that if we create a logical structure ( such as an or ) -- each literal references a unique identifier from the right. Otherwise,
* subsequent visitors will reference incorrection sub trees, and potentially negate the activity of the query model visitor
*/
Set<List<JexlNode>> newSet = product.stream().map(list -> list.stream().map(RebuildingVisitor::copy).collect(Collectors.toList())).collect(Collectors.toSet());
if (product.size() > 1) {
JexlNode expanded;
if (requiresAnd) {
expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.AND_NODE, node, newSet);
} else {
expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.OR_NODE, node, newSet);
}
if (log.isTraceEnabled())
log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
return expanded;
} else if (1 == product.size()) {
List<JexlNode> pair = product.iterator().next();
JexlNode expanded = JexlNodeFactory.buildUntypedBinaryNode(node, pair.get(0), pair.get(1));
if (log.isTraceEnabled())
log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
return expanded;
}
// If we couldn't map anything, return a copy
if (log.isTraceEnabled())
log.trace("just returning the original:" + PrintingVisitor.formattedQueryString(node));
return node;
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class LoadModelFromXml method loadModelFromXml.
public static QueryModel loadModelFromXml(InputStream stream) throws Exception {
SAXParserFactory spf = SAXParserFactory.newInstance();
spf.setFeature("http://xml.org/sax/features/external-general-entities", false);
spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
spf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
spf.setNamespaceAware(true);
Source xmlSource = new SAXSource(spf.newSAXParser().getXMLReader(), new InputSource(stream));
JAXBContext ctx = JAXBContext.newInstance(Model.class);
Unmarshaller um = ctx.createUnmarshaller();
Model xmlModel = (Model) um.unmarshal(xmlSource);
if (log.isDebugEnabled()) {
log.debug(xmlModel.getName());
for (FieldMapping fieldMapping : xmlModel.getFields()) {
log.debug(fieldMapping.toString());
}
}
QueryModel model = new QueryModel();
for (FieldMapping mapping : xmlModel.getFields()) {
switch(mapping.getDirection()) {
case FORWARD:
model.addTermToModel(mapping.getModelFieldName(), mapping.getFieldName());
break;
case REVERSE:
model.addTermToReverseModel(mapping.getFieldName(), mapping.getModelFieldName());
break;
default:
log.error("Unknown direction: " + mapping.getDirection());
}
}
if (model.getForwardQueryMapping().isEmpty() && model.getReverseQueryMapping().isEmpty()) {
throw new IllegalArgumentException("The resulting, loaded query model was empty.");
}
return model;
}
use of datawave.query.model.QueryModel in project datawave by NationalSecurityAgency.
the class MetadataHelperQueryModelProvider method getQueryModel.
@Override
public QueryModel getQueryModel() {
QueryModel queryModel = null;
if (config.getQueryModel() != null) {
log.debug("Using a cached query model");
queryModel = config.getQueryModel();
} else if (null != config.getModelName() && null != config.getModelTableName()) {
log.debug("Generating a query model");
try {
queryModel = metadataHelper.getQueryModel(config.getModelTableName(), config.getModelName(), config.getUnevaluatedFields(), config.getDatatypeFilter());
config.setQueryModel(queryModel);
} catch (TableNotFoundException e) {
QueryException qe = new QueryException(DatawaveErrorCode.QUERY_MODEL_FETCH_ERROR, e);
log.error(qe);
throw new DatawaveFatalQueryException(qe);
}
if (log.isTraceEnabled()) {
log.trace("forward queryModel: " + queryModel.getForwardQueryMapping());
log.trace("reverse queryModel: " + queryModel.getReverseQueryMapping());
}
}
return queryModel;
}
Aggregations