Examples with DatawaveFatalQueryException datawave.query.exceptions.DatawaveFatalQueryException used on opensource projects

Example 11 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class PushdownMissingIndexRangeNodesVisitor method missingIndexRange.

private boolean missingIndexRange(ASTERNode node) {
    Object literal = JexlASTHelper.getLiteralValue(node);
    if (literal != null) {
        String strLiteral = String.valueOf(literal);
        JavaRegexAnalyzer analyzer = null;
        try {
            analyzer = new JavaRegexAnalyzer(strLiteral);
            if (analyzer.isLeadingLiteral()) {
                String leadingLiteral = analyzer.getLeadingLiteral();
                StringBuilder endRange = new StringBuilder().append(leadingLiteral);
                char lastChar = leadingLiteral.charAt(leadingLiteral.length() - 1);
                if (lastChar < Character.MAX_VALUE) {
                    lastChar++;
                    endRange.setCharAt(endRange.length() - 1, lastChar);
                } else {
                    endRange.append((char) 0);
                }
                for (IndexHole hole : indexHoles) {
                    if (hole.overlaps(this.beginDate, this.endDate, leadingLiteral, endRange.toString())) {
                        return true;
                    } else if (hole.after(strLiteral)) {
                        return false;
                    }
                }
            }
        } catch (JavaRegexAnalyzer.JavaRegexParseException e) {
            log.error("Unable to parse regex " + strLiteral, e);
            throw new DatawaveFatalQueryException("Unable to parse regex " + strLiteral, e);
        }
    }
    return false;
}

Example 12 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class QueryModelVisitor method expandBinaryNodeFromModel.

/**
 * Applies the forward mapping from the QueryModel to a node, expanding the node into an Or if needed.
 *
 * @param node
 * @param data
 * @return
 */
protected JexlNode expandBinaryNodeFromModel(JexlNode node, Object data) {
    String field = JexlASTHelper.getIdentifier(node);
    if (isFieldExcluded(field)) {
        return node;
    }
    // Count the immediate children:
    int childCount = node.jjtGetNumChildren();
    if (childCount != 2) {
        QueryException qe = new QueryException(DatawaveErrorCode.BINARY_NODE_TOO_MANY_CHILDREN, MessageFormat.format("Node: {0}", PrintingVisitor.formattedQueryString(node)));
        throw new DatawaveFatalQueryException(qe);
    }
    // Find identifiers
    List<ASTIdentifier> allidentifiers = JexlASTHelper.getIdentifiers(node);
    // If we don't have any identifiers, we have nothing to expand
    if (allidentifiers.isEmpty()) {
        return node;
    }
    JexlNode leftNode = node.jjtGetChild(0);
    JexlNode rightNode = node.jjtGetChild(1);
    if (log.isTraceEnabled()) {
        log.trace("leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // this will expand identifiers that have a method connected to them
    boolean leftState = JexlASTHelper.HasMethodVisitor.hasMethod(leftNode);
    if (leftState) {
        // there is a method under leftNode
        leftNode = (JexlNode) leftNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    boolean rightState = JexlASTHelper.HasMethodVisitor.hasMethod(rightNode);
    if (rightState) {
        // there is a method under rightNode
        rightNode = (JexlNode) rightNode.jjtAccept(this.simpleQueryModelVisitor, null);
    }
    // expand any identifiers inside of methods/functions in the left and right nodes
    leftNode = (JexlNode) leftNode.jjtAccept(this, null);
    rightNode = (JexlNode) rightNode.jjtAccept(this, null);
    if (log.isTraceEnabled()) {
        log.trace("after expansion, leftNode:" + PrintingVisitor.formattedQueryString(leftNode));
        log.trace("after expansion, leftNodeQuery:" + JexlStringBuildingVisitor.buildQuery(leftNode));
        log.trace("after expansion, rightNode:" + PrintingVisitor.formattedQueryString(rightNode));
        log.trace("after expansion, rightNodeQuery:" + JexlStringBuildingVisitor.buildQuery(rightNode));
    }
    // if state == true on either side, then there is a method on one side and we are done applying the model
    if (leftState || rightState) {
        JexlNode toReturn = JexlNodeFactory.buildUntypedBinaryNode(node, leftNode, rightNode);
        if (log.isTraceEnabled()) {
            log.trace("done early. returning:" + JexlStringBuildingVisitor.buildQuery(toReturn));
        }
        return toReturn;
    }
    JexlNode leftSeed, rightSeed;
    Set<JexlNode> left = Sets.newHashSet(), right = Sets.newHashSet();
    boolean isNullEquality = false;
    if (node instanceof ASTEQNode && (leftNode instanceof ASTNullLiteral || rightNode instanceof ASTNullLiteral)) {
        isNullEquality = true;
    }
    // the query has been previously groomed so that identifiers are on the left and literals are on the right
    // an identifier with a method attached will have already been substituted above (and will return null for the IdentifierOpLiteral)
    // The normal case of `IDENTIFIER op 'literal'`
    JexlASTHelper.IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op != null) {
        // One identifier
        leftSeed = op.getIdentifier();
        rightSeed = op.getLiteral();
        if (rightSeed instanceof ASTNullLiteral && node instanceof ASTEQNode) {
            isNullEquality = true;
        }
    } else {
        // We know from above that childCount == 2. We may have a reference on both sides of the expression
        leftSeed = node.jjtGetChild(0);
        rightSeed = node.jjtGetChild(1);
    }
    if (leftSeed instanceof ASTReference) {
        // String fieldName = JexlASTHelper.getIdentifier((JexlNode)leftSeed);
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(leftSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(leftSeed) + " from " + JexlStringBuildingVisitor.buildQuery(leftNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                left.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (leftSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) leftSeed))) {
            left.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        left.add(leftSeed);
    }
    if (rightSeed instanceof ASTReference) {
        List<ASTIdentifier> identifiers = JexlASTHelper.getIdentifiers(rightSeed);
        if (identifiers.size() > 1) {
            log.warn("I did not expect to see more than one Identifier here for " + JexlStringBuildingVisitor.buildQuery(rightSeed) + " from " + JexlStringBuildingVisitor.buildQuery(rightNode));
        }
        for (ASTIdentifier identifier : identifiers) {
            for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier(identifier))) {
                right.add(JexlNodeFactory.buildIdentifier(fieldName));
            }
        }
    } else if (rightSeed instanceof ASTIdentifier) {
        for (String fieldName : getAliasesForField(JexlASTHelper.deconstructIdentifier((ASTIdentifier) rightSeed))) {
            right.add(JexlNodeFactory.buildIdentifier(fieldName));
        }
    } else {
        // Not an identifier, therefore it's probably a literal
        right.add(rightSeed);
    }
    boolean requiresAnd = isNullEquality || node instanceof ASTNENode;
    @SuppressWarnings("unchecked") Set<List<JexlNode>> // retrieve the cartesian product
    product = Sets.cartesianProduct(left, right);
    /**
     * use the product transformer to shallow copy the jexl nodes. We've created new nodes that will be embedded within an ast reference. As a result, we
     * need to ensure that if we create a logical structure ( such as an or ) -- each literal references a unique identifier from the right. Otherwise,
     * subsequent visitors will reference incorrection sub trees, and potentially negate the activity of the query model visitor
     */
    Set<List<JexlNode>> newSet = product.stream().map(list -> list.stream().map(RebuildingVisitor::copy).collect(Collectors.toList())).collect(Collectors.toSet());
    if (product.size() > 1) {
        JexlNode expanded;
        if (requiresAnd) {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.AND_NODE, node, newSet);
        } else {
            expanded = JexlNodeFactory.createNodeTreeFromPairs(ContainerType.OR_NODE, node, newSet);
        }
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    } else if (1 == product.size()) {
        List<JexlNode> pair = product.iterator().next();
        JexlNode expanded = JexlNodeFactory.buildUntypedBinaryNode(node, pair.get(0), pair.get(1));
        if (log.isTraceEnabled())
            log.trace("expanded:" + PrintingVisitor.formattedQueryString(expanded));
        return expanded;
    }
    // If we couldn't map anything, return a copy
    if (log.isTraceEnabled())
        log.trace("just returning the original:" + PrintingVisitor.formattedQueryString(node));
    return node;
}

Example 13 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class MetadataHelperQueryModelProvider method getQueryModel.

@Override
public QueryModel getQueryModel() {
    QueryModel queryModel = null;
    if (config.getQueryModel() != null) {
        log.debug("Using a cached query model");
        queryModel = config.getQueryModel();
    } else if (null != config.getModelName() && null != config.getModelTableName()) {
        log.debug("Generating a query model");
        try {
            queryModel = metadataHelper.getQueryModel(config.getModelTableName(), config.getModelName(), config.getUnevaluatedFields(), config.getDatatypeFilter());
            config.setQueryModel(queryModel);
        } catch (TableNotFoundException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.QUERY_MODEL_FETCH_ERROR, e);
            log.error(qe);
            throw new DatawaveFatalQueryException(qe);
        }
        if (log.isTraceEnabled()) {
            log.trace("forward queryModel: " + queryModel.getForwardQueryMapping());
            log.trace("reverse queryModel: " + queryModel.getReverseQueryMapping());
        }
    }
    return queryModel;
}

Example 14 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class ShardLimitingIterator method next.

@Override
public Entry<Key, Value> next() {
    Entry<Key, Value> top = currentQueue.poll();
    if (currentQueue.size() >= maxShardsPerDay) {
        Key topKey = top.getKey();
        if (log.isTraceEnabled())
            log.trace(topKey + " for " + currentDay + " exceeds limit of " + maxShardsPerDay + " with " + currentQueue.size());
        Key newKey = new Key(topKey.getRow(), topKey.getColumnFamily(), new Text(currentDay), topKey.getColumnVisibility(), topKey.getTimestamp());
        currentQueue.clear();
        IndexInfo info = new IndexInfo(-1);
        Value newValue = null;
        try {
            ByteArrayOutputStream outByteStream = new ByteArrayOutputStream();
            DataOutputStream outDataStream = new DataOutputStream(outByteStream);
            info.write(outDataStream);
            outDataStream.close();
            outByteStream.close();
            newValue = new Value(outByteStream.toByteArray());
        } catch (IOException e) {
            throw new DatawaveFatalQueryException(e);
        }
        return Maps.immutableEntry(newKey, newValue);
    } else {
        if (log.isTraceEnabled())
            log.trace(top + " for " + currentDay + " does not exceed limit of " + maxShardsPerDay + " with " + currentQueue.size());
        return top;
    }
}

Example 15 with DatawaveFatalQueryException

use of datawave.query.exceptions.DatawaveFatalQueryException in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitor method buildExceededFromTermFrequency.

/**
 * @param data
 */
private NestedIterator<Key> buildExceededFromTermFrequency(String identifier, JexlNode rootNode, JexlNode sourceNode, LiteralRange<?> range, Object data) {
    if (limitLookup) {
        ChainableEventDataQueryFilter wrapped = createWrappedTermFrequencyFilter(identifier, sourceNode, attrFilter);
        NestedIterator<Key> eventFieldIterator = new EventFieldIterator(rangeLimiter, source.deepCopy(env), identifier, new AttributeFactory(this.typeMetadata), getEventFieldAggregator(identifier, wrapped));
        TermFrequencyIndexBuilder builder = new TermFrequencyIndexBuilder();
        builder.setSource(source.deepCopy(env));
        builder.setTypeMetadata(typeMetadata);
        builder.setFieldsToAggregate(fieldsToAggregate);
        builder.setTimeFilter(timeFilter);
        builder.setAttrFilter(attrFilter);
        builder.setDatatypeFilter(datatypeFilter);
        builder.setEnv(env);
        builder.setTermFrequencyAggregator(getTermFrequencyAggregator(identifier, sourceNode, attrFilter, attrFilter != null ? attrFilter.getMaxNextCount() : -1));
        builder.setNode(rootNode);
        Range fiRange = getFiRangeForTF(range);
        builder.setRange(fiRange);
        builder.setField(identifier);
        NestedIterator<Key> tfIterator = builder.build();
        OrIterator tfMerge = new OrIterator(Arrays.asList(tfIterator, eventFieldIterator));
        return tfMerge;
    } else {
        QueryException qe = new QueryException(DatawaveErrorCode.UNEXPECTED_SOURCE_NODE, MessageFormat.format("{0}", "buildExceededFromTermFrequency"));
        throw new DatawaveFatalQueryException(qe);
    }
}