Examples with IteratorBuildingVisitor datawave.query.jexl.visitors.IteratorBuildingVisitor used on opensource projects

Example 1 with IteratorBuildingVisitor

use of datawave.query.jexl.visitors.IteratorBuildingVisitor in project datawave by NationalSecurityAgency.

the class FieldIndexOnlyQueryIterator method createAndSeekIndexIterator.

protected void createAndSeekIndexIterator(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException, ConfigException, IllegalAccessException, InstantiationException {
    boolean isQueryFullySatisfiedInitialState = true;
    String hitListOptionString = documentOptions.get("hit.list");
    if (hitListOptionString != null) {
        boolean hitListOption = Boolean.parseBoolean(hitListOptionString);
        if (hitListOption) {
            // if hit list is on, don't attempt satisfiability
            isQueryFullySatisfiedInitialState = false;
        // don't even make a SatisfactionVisitor.....
        }
    }
    Collection<String> unindexedTypes = Lists.newArrayList();
    Set<String> keys = fetchDataTypeKeys(this.documentOptions.get(NON_INDEXED_DATATYPES));
    String compressedOptionString = this.documentOptions.get(QUERY_MAPPING_COMPRESS);
    if (!org.apache.commons.lang3.StringUtils.isBlank(compressedOptionString)) {
        boolean compressedOption = Boolean.parseBoolean(compressedOptionString);
        if (compressedOption) {
            for (String key : keys) {
                unindexedTypes.add(decompressOption(key, QueryOptions.UTF8));
            }
        }
    } else {
        unindexedTypes.addAll(keys);
    }
    if (isQueryFullySatisfiedInitialState) {
        // we'll charge in with optimism
        SatisfactionVisitor satisfactionVisitor = this.createSatisfiabilityVisitor(true);
        satisfactionVisitor.setUnindexedFields(unindexedTypes);
        // visit() and get the root which is the root of a tree of Boolean Logic Iterator<Key>'s
        this.script.jjtAccept(satisfactionVisitor, null);
        isQueryFullySatisfiedInitialState = satisfactionVisitor.isQueryFullySatisfied();
    }
    IteratorBuildingVisitor visitor = createIteratorBuildingVisitor(null, isQueryFullySatisfiedInitialState, sortedUIDs);
    visitor.setUnindexedFields(unindexedTypes);
    // visit() and get the root which is the root of a tree of Boolean Logic Iterator<Key>'s
    script.jjtAccept(visitor, null);
    NestedIterator<Key> root = visitor.root();
    if (null == root) {
        throw new UnindexedException("Could not instantiate iterators over field index for " + this.getQuery());
    } else {
        this.fieldIndexResults = new AccumuloFieldIndexIterable(root);
    }
    // Otherwise, we have to use the field index
    // Seek() the boolean logic stuff
    this.fieldIndexResults.seek(range, columnFamilies, inclusive);
}

Example 2 with IteratorBuildingVisitor

use of datawave.query.jexl.visitors.IteratorBuildingVisitor in project datawave by NationalSecurityAgency.

the class QueryIterator method getOrSetKeySource.

protected NestedIterator<Key> getOrSetKeySource(final Range documentRange, ASTJexlScript rangeScript) throws IOException, ConfigException, IllegalAccessException, InstantiationException {
    NestedIterator<Key> sourceIter = null;
    // query)
    if (!this.isFullTableScanOnly()) {
        boolean isQueryFullySatisfiedInitialState = batchedQueries <= 0;
        String hitListOptionString = documentOptions.get(QueryOptions.HIT_LIST);
        if (hitListOptionString != null) {
            boolean hitListOption = Boolean.parseBoolean(hitListOptionString);
            if (hitListOption) {
                // if hit
                isQueryFullySatisfiedInitialState = false;
            // list is
            // on, don't
            // attempt
            // satisfiability
            // don't even make a SatisfactionVisitor.....
            }
        }
        if (isQueryFullySatisfiedInitialState) {
            // we'll
            SatisfactionVisitor satisfactionVisitor = this.createSatisfiabilityVisitor(true);
            // charge
            // in
            // with
            // optimism
            // visit() and get the root which is the root of a tree of
            // Boolean Logic Iterator<Key>'s
            rangeScript.jjtAccept(satisfactionVisitor, null);
            isQueryFullySatisfiedInitialState = satisfactionVisitor.isQueryFullySatisfied();
        }
        IteratorBuildingVisitor visitor = createIteratorBuildingVisitor(documentRange, isQueryFullySatisfiedInitialState, this.sortedUIDs);
        // visit() and get the root which is the root of a tree of
        // Boolean Logic Iterator<Key>'s
        rangeScript.jjtAccept(visitor, null);
        sourceIter = visitor.root();
        if (visitor.isQueryFullySatisfied()) {
            this.fieldIndexSatisfiesQuery = true;
        }
        // Print out the boolean logic tree of iterators
        debugBooleanLogicIterators(sourceIter);
        if (sourceIter != null) {
            sourceIter = new SeekableNestedIterator(sourceIter, this.myEnvironment);
        }
    }
    // resort to a full table scan otherwise
    if (sourceIter == null) {
        sourceIter = getEventDataNestedIterator(source);
    }
    return sourceIter;
}

Example 3 with IteratorBuildingVisitor

use of datawave.query.jexl.visitors.IteratorBuildingVisitor in project datawave by NationalSecurityAgency.

the class QueryIterator method getEvaluation.

protected Iterator<Entry<Key, Document>> getEvaluation(NestedQueryIterator<Key> documentSource, SortedKeyValueIterator<Key, Value> sourceDeepCopy, Iterator<Entry<Key, Document>> documents, CompositeMetadata compositeMetadata, TypeMetadata typeMetadataForEval, Collection<ByteSequence> columnFamilies, boolean inclusive) {
    // Filter the Documents by testing them against the JEXL query
    if (!this.disableEvaluation) {
        JexlEvaluation jexlEvaluationFunction = getJexlEvaluation(documentSource);
        Collection<String> variables = null;
        if (null != documentSource && null != documentSource.getQuery()) {
            variables = VariableNameVisitor.parseQuery(jexlEvaluationFunction.parse(documentSource.getQuery()));
        } else {
            variables = VariableNameVisitor.parseQuery(jexlEvaluationFunction.parse(query));
        }
        final Iterator<Tuple2<Key, Document>> tupleItr = Iterators.transform(documents, new EntryToTuple<>());
        // get the function we use for the tf functionality. Note we are
        // getting an additional source deep copy for this function
        final Iterator<Tuple3<Key, Document, Map<String, Object>>> itrWithContext;
        if (this.isTermFrequenciesRequired()) {
            // The TFFunction can only prune non index-only fields
            Set<String> tfIndexOnlyFields = Sets.intersection(getTermFrequencyFields(), getIndexOnlyFields());
            Function<Tuple2<Key, Document>, Tuple3<Key, Document, Map<String, Object>>> tfFunction;
            tfFunction = TFFactory.getFunction(getScript(documentSource), getContentExpansionFields(), getTermFrequencyFields(), this.getTypeMetadata(), super.equality, getEvaluationFilter(), sourceDeepCopy.deepCopy(myEnvironment), tfIndexOnlyFields);
            itrWithContext = TraceIterators.transform(tupleItr, tfFunction, "Term Frequency Lookup");
        } else {
            itrWithContext = Iterators.transform(tupleItr, new EmptyContext<>());
        }
        try {
            IteratorBuildingVisitor iteratorBuildingVisitor = createIteratorBuildingVisitor(getDocumentRange(documentSource), false, this.sortedUIDs);
            Multimap<String, JexlNode> delayedNonEventFieldMap = DelayedNonEventSubTreeVisitor.getDelayedNonEventFieldMap(iteratorBuildingVisitor, script, getNonEventFields());
            IndexOnlyContextCreatorBuilder contextCreatorBuilder = new IndexOnlyContextCreatorBuilder().setSource(sourceDeepCopy).setRange(getDocumentRange(documentSource)).setTypeMetadata(typeMetadataForEval).setCompositeMetadata(compositeMetadata).setOptions(this).setVariables(variables).setIteratorBuildingVisitor(iteratorBuildingVisitor).setDelayedNonEventFieldMap(delayedNonEventFieldMap).setEquality(equality).setColumnFamilies(columnFamilies).setInclusive(inclusive).setComparatorFactory(this);
            final IndexOnlyContextCreator contextCreator = contextCreatorBuilder.build();
            if (exceededOrEvaluationCache != null) {
                contextCreator.addAdditionalEntries(exceededOrEvaluationCache);
            }
            final Iterator<Tuple3<Key, Document, DatawaveJexlContext>> itrWithDatawaveJexlContext = Iterators.transform(itrWithContext, contextCreator);
            Iterator<Tuple3<Key, Document, DatawaveJexlContext>> matchedDocuments = statelessFilter(itrWithDatawaveJexlContext, jexlEvaluationFunction);
            if (log.isTraceEnabled()) {
                log.trace("arithmetic:" + arithmetic + " range:" + getDocumentRange(documentSource) + ", thread:" + Thread.currentThread());
            }
            return Iterators.transform(matchedDocuments, new TupleToEntry<>());
        } catch (InstantiationException | MalformedURLException | IllegalAccessException | ConfigException e) {
            throw new IllegalStateException("Could not perform delayed index only evaluation", e);
        }
    } else if (log.isTraceEnabled()) {
        log.trace("Evaluation is disabled, not instantiating Jexl evaluation logic");
    }
    return documents;
}