Examples with IdentifierOpLiteral datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral used on opensource projects

Example 1 with IdentifierOpLiteral

use of datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral in project datawave by NationalSecurityAgency.

the class RangeStream method visit.

@Override
public Object visit(ASTERNode node, Object data) {
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op == null) {
        return ScannerStream.unindexed(node);
    }
    final String fieldName = op.deconstructIdentifier();
    // HACK to make EVENT_DATATYPE queries work
    if (QueryOptions.DEFAULT_DATATYPE_FIELDNAME.equals(fieldName)) {
        return ScannerStream.unindexed(node);
    }
    if (isUnOrNotFielded(node)) {
        return ScannerStream.noData(node);
    }
    if (isUnindexed(node)) {
        return ScannerStream.unindexed(node);
    }
    if (node instanceof ASTUnknownFieldERNode) {
        return ScannerStream.unknownField(node);
    }
    return ScannerStream.noData(node);
}

Example 2 with IdentifierOpLiteral

use of datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral in project datawave by NationalSecurityAgency.

the class ExpandMultiNormalizedTerms method expandNodeForNormalizers.

/**
 * @param node
 * @param data
 * @return
 */
protected JexlNode expandNodeForNormalizers(JexlNode node, Object data) {
    JexlNode nodeToReturn = node;
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op != null) {
        final String fieldName = op.deconstructIdentifier();
        final Object literal = op.getLiteralValue();
        // Get all of the indexed or normalized dataTypes for the field name
        Set<Type<?>> dataTypes = Sets.newHashSet(config.getQueryFieldsDatatypes().get(fieldName));
        dataTypes.addAll(config.getNormalizedFieldsDatatypes().get(fieldName));
        // Catch the case of the user entering FIELD == null
        if (!dataTypes.isEmpty() && null != literal) {
            try {
                String term = literal.toString();
                Set<String> normalizedTerms = Sets.newHashSet();
                // Build up a set of normalized terms using each normalizer
                for (Type<?> normalizer : dataTypes) {
                    try {
                        if (node instanceof ASTNRNode || node instanceof ASTERNode) {
                            normalizedTerms.add(normalizer.normalizeRegex(term));
                        } else {
                            normalizedTerms.add(normalizer.normalize(term));
                        }
                        log.debug("normalizedTerms=" + normalizedTerms);
                    } catch (IpAddressNormalizer.Exception ipex) {
                        try {
                            String[] lowHi = ((IpAddressType) normalizer).normalizeCidrToRange(term);
                            // node was FIELD == 'cidr'
                            // change to FIELD >= low and FIELD <= hi
                            JexlNode geNode = JexlNodeFactory.buildNode(new ASTGENode(ParserTreeConstants.JJTGENODE), fieldName, lowHi[0]);
                            JexlNode leNode = JexlNodeFactory.buildNode(new ASTLENode(ParserTreeConstants.JJTLENODE), fieldName, lowHi[1]);
                            // now link em up
                            return BoundedRange.create(JexlNodeFactory.createAndNode(Arrays.asList(geNode, leNode)));
                        } catch (Exception ex) {
                            if (log.isTraceEnabled()) {
                                log.trace("Could not normalize " + term + " as cidr notation with: " + normalizer.getClass());
                            }
                        }
                    // this could be CIDR notation, attempt to expand the node to the cidr range
                    } catch (Exception ne) {
                        if (log.isTraceEnabled()) {
                            log.trace("Could not normalize " + term + " using " + normalizer.getClass());
                        }
                    }
                }
                if (normalizedTerms.size() > 1) {
                    nodeToReturn = JexlNodeFactory.createNodeTreeFromFieldValues(ContainerType.OR_NODE, node, node, fieldName, normalizedTerms);
                } else if (1 == normalizedTerms.size()) {
                    // If there is only one term, we don't need to make an OR
                    nodeToReturn = JexlNodeFactory.buildUntypedNewLiteralNode(node, fieldName, normalizedTerms.iterator().next());
                } else {
                    // If we couldn't map anything, return a copy
                    nodeToReturn = JexlNodeFactory.buildUntypedNewLiteralNode(node, fieldName, literal);
                }
            } catch (Exception e) {
                QueryException qe = new QueryException(DatawaveErrorCode.NODE_EXPANSION_ERROR, e, MessageFormat.format("Node: {0}, Datatypes: {1}", PrintingVisitor.formattedQueryString(node), dataTypes));
                log.error(qe);
                throw new DatawaveFatalQueryException(qe);
            }
        }
    }
    return nodeToReturn;
}

Example 3 with IdentifierOpLiteral

use of datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral in project datawave by NationalSecurityAgency.

the class FixUnindexedNumericTerms method expandNodeForNormalizers.

protected JexlNode expandNodeForNormalizers(JexlNode node) {
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op == null) {
        return copy(node);
    }
    final String fieldName = op.deconstructIdentifier();
    Object literal = op.getLiteralValue();
    // Get all the normalizers for the field name
    Collection<Type<?>> normalizers = config.getQueryFieldsDatatypes().get(fieldName);
    // Catch the case of the user entering FIELD == null
    if (normalizers.isEmpty() && literal instanceof String) {
        try {
            literal = NumberUtils.createNumber((String) literal);
        } catch (Exception nfe) {
        // leave literal as is
        }
    }
    // Return a copy of the node with a potentially converted literal
    JexlNode copy = copy(node);
    return JexlNodeFactory.buildUntypedNewLiteralNode(copy, fieldName, literal);
}

Example 4 with IdentifierOpLiteral

use of datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitor method createIndexOnlyKey.

protected SortedKeyValueIterator<Key, Value> createIndexOnlyKey(ASTEQNode node) throws IOException {
    Key newStartKey = getKey(node);
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (null == op || null == op.getLiteralValue()) {
        // deep copy since this is likely a null literal
        return source.deepCopy(env);
    }
    String fn = op.deconstructIdentifier();
    String literal = String.valueOf(op.getLiteralValue());
    if (log.isTraceEnabled()) {
        log.trace("createIndexOnlyKey for " + fn + " " + literal + " " + newStartKey);
    }
    List<Entry<Key, Value>> kv = Lists.newArrayList();
    if (null != limitedMap.get(Maps.immutableEntry(fn, literal))) {
        kv.add(limitedMap.get(Maps.immutableEntry(fn, literal)));
    } else {
        SortedKeyValueIterator<Key, Value> mySource = limitedSource;
        // if source size > 0, we are free to use up to that number for this query
        if (source.getSourceSize() > 0)
            mySource = source.deepCopy(env);
        mySource.seek(new Range(newStartKey, true, newStartKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false), Collections.emptyList(), false);
        if (mySource.hasTop()) {
            kv.add(Maps.immutableEntry(mySource.getTopKey(), Constants.NULL_VALUE));
        }
    }
    return new IteratorToSortedKeyValueIterator(kv.iterator());
}

Example 5 with IdentifierOpLiteral

use of datawave.query.jexl.JexlASTHelper.IdentifierOpLiteral in project datawave by NationalSecurityAgency.

the class RangeStream method visit.

@Override
public ScannerStream visit(ASTEQNode node, Object data) {
    if (isUnOrNotFielded(node)) {
        return ScannerStream.noData(node);
    }
    // We are looking for identifier = literal
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (op == null) {
        return ScannerStream.unindexed(node);
    }
    final String fieldName = op.deconstructIdentifier();
    // Null literals cannot be resolved against the index.
    if (op.getLiteralValue() == null) {
        return ScannerStream.unindexed(node);
    }
    // toString of String returns the String
    String literal = op.getLiteralValue().toString();
    if (QueryOptions.DEFAULT_DATATYPE_FIELDNAME.equals(fieldName)) {
        return ScannerStream.unindexed(node);
    }
    // Check if field is not indexed
    if (!isIndexed(fieldName, config.getIndexedFields())) {
        try {
            if (this.getAllFieldsFromHelper().contains(fieldName)) {
                log.debug("{\"" + fieldName + "\": \"" + literal + "\"} is not indexed.");
                return ScannerStream.unindexed(node);
            }
        } catch (TableNotFoundException e) {
            log.error(e);
            throw new RuntimeException(e);
        }
        log.debug("{\"" + fieldName + "\": \"" + literal + "\"} is not an observed field.");
        return ScannerStream.unknownField(node);
    }
    // Final case, field is indexed
    log.debug("\"" + fieldName + "\" is indexed. for " + literal);
    try {
        int stackStart = config.getBaseIteratorPriority();
        RangeStreamScanner scannerSession;
        SessionOptions options = new SessionOptions();
        options.fetchColumnFamily(new Text(fieldName));
        options.addScanIterator(makeDataTypeFilter(config, stackStart++));
        final IteratorSetting uidSetting;
        // Create the range for the term from the provided config.
        Range range = rangeForTerm(literal, fieldName, config);
        if (limitScanners) {
            // Setup the CreateUidsIterator
            scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), config.getShardsPerDayThreshold());
            uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass);
            uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.valueOf(collapseUids).toString());
            uidSetting.addOption(CreateUidsIterator.PARSE_TLD_UIDS, Boolean.valueOf(config.getParseTldUids()).toString());
        } else {
            // Setup so this is a pass-through
            scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), config.getShardsPerDayThreshold());
            uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass);
            uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.valueOf(false).toString());
            uidSetting.addOption(CreateUidsIterator.PARSE_TLD_UIDS, Boolean.valueOf(false).toString());
        }
        /*
             * Create a scanner in the initialized state so that we can scan immediately
             */
        if (log.isTraceEnabled()) {
            log.trace("Building delayed scanner for " + fieldName + ", literal= " + literal);
        }
        // Configure common settings on the ScannerSession
        options.addScanIterator(uidSetting);
        String queryString = fieldName + "=='" + literal + "'";
        options.addScanIterator(QueryScannerHelper.getQueryInfoIterator(config.getQuery(), false, queryString));
        scannerSession.setOptions(options);
        scannerSession.setMaxResults(config.getMaxIndexBatchSize());
        scannerSession.setExecutor(streamExecutor);
        scannerSession.setRanges(Collections.singleton(range));
        // Create the EntryParser prior to ScannerStream.
        EntryParser entryParser = new EntryParser(node, fieldName, literal, indexOnlyFields);
        return ScannerStream.initialized(scannerSession, entryParser, node);
    } catch (Exception e) {
        log.error(e);
        throw new RuntimeException(e);
    }
}