Examples with NestedIterator datawave.query.iterator.NestedIterator used on opensource projects

Example 6 with NestedIterator

use of datawave.query.iterator.NestedIterator in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitorTest method visitNeNode_nullValueTest.

/**
 * null value should result in no iterator being built. In this case a top level negation is not allowed, so pair it with an indexed lookup
 */
@Test
public void visitNeNode_nullValueTest() throws ParseException {
    ASTJexlScript query = JexlASTHelper.parseJexlQuery("FOO == 'bar' && FIELD != null");
    IteratorBuildingVisitor visitor = getDefault();
    query.jjtAccept(visitor, null);
    NestedIterator nestedIterator = visitor.root();
    // the only leaf in the iterator is FOO == 'bar'
    Assert.assertNotEquals(null, nestedIterator);
    Assert.assertEquals(1, nestedIterator.leaves().size());
    Assert.assertTrue(nestedIterator.leaves().iterator().next().toString().contains("FOO"));
}

Example 7 with NestedIterator

use of datawave.query.iterator.NestedIterator in project datawave by NationalSecurityAgency.

the class IteratorBuildingVisitorTest method eval.

private void eval(ASTJexlScript query, Range docRange, Key docKeyHit, List<Map.Entry<Key, Value>> source, boolean buildDoc, Map<String, List<String>> docKeys, Set<String> termFrequencyFields, Set<String> aggregationFields, Set<String> indexOnlyFields) throws IOException {
    IteratorBuildingVisitor visitor = new IteratorBuildingVisitor();
    TypeMetadata typeMetadata = new TypeMetadata();
    Iterator<Map.Entry<Key, Value>> iterator = source.iterator();
    IteratorEnvironment env = new BaseIteratorEnvironment();
    visitor.setSource(new SourceFactory(iterator), env);
    // configure the visitor for use
    visitor.setTermFrequencyFields(termFrequencyFields);
    visitor.setFieldsToAggregate(aggregationFields);
    visitor.setIndexOnlyFields(indexOnlyFields);
    visitor.setRange(docRange);
    visitor.setTimeFilter(TimeFilter.alwaysTrue());
    visitor.setLimitLookup(true);
    visitor.setTypeMetadata(typeMetadata);
    query.jjtAccept(visitor, null);
    NestedIterator result = visitor.root();
    Assert.assertTrue(result != null);
    SeekableNestedIterator seekableNestedIterator = new SeekableNestedIterator(result, env);
    seekableNestedIterator.seek(docRange, null, true);
    seekableNestedIterator.initialize();
    // asserts for a hit or miss
    if (docKeyHit == null) {
        Assert.assertFalse(seekableNestedIterator.hasNext());
    } else {
        Assert.assertTrue(seekableNestedIterator.hasNext());
        Key next = (Key) seekableNestedIterator.next();
        Assert.assertTrue(next != null);
        Assert.assertTrue(next.getRow().toString().equals(docKeyHit.getRow().toString()));
        Assert.assertTrue(next.getColumnFamily().toString().equals(docKeyHit.getColumnFamily().toString()));
        // asserts for document build
        Document d = seekableNestedIterator.document();
        Assert.assertTrue(d != null);
        if (buildDoc) {
            // +1 is for RECORD_ID field
            Assert.assertTrue(docKeys.keySet().size() + 1 == d.getDictionary().size());
            // verify hits for each specified field
            for (String field : docKeys.keySet()) {
                List<String> expected = docKeys.get(field);
                if (expected.size() == 1) {
                    // verify the only doc
                    Assert.assertTrue(d.getDictionary().get(field).getData().equals(expected.get(0)));
                } else {
                    // the data should be a set, verify it matches expected
                    Object dictData = d.getDictionary().get(field).getData();
                    Assert.assertTrue(dictData != null);
                    Assert.assertTrue(dictData instanceof Set);
                    Set dictSet = (Set) dictData;
                    Assert.assertTrue(dictSet.size() == expected.size());
                    Iterator<Attribute> dictIterator = dictSet.iterator();
                    while (dictIterator.hasNext()) {
                        Assert.assertTrue(expected.remove(dictIterator.next().getData()));
                    }
                    // verify that the expected set is now empty
                    Assert.assertTrue(expected.size() == 0);
                }
            }
        } else {
            // doc should be empty
            Assert.assertTrue(d.getDictionary().size() == 0);
        }
        // there should be no other hits
        Assert.assertFalse(seekableNestedIterator.hasNext());
    }
}

Example 8 with NestedIterator

use of datawave.query.iterator.NestedIterator in project datawave by NationalSecurityAgency.

the class DelayedNonEventIndexContext method fetchOnDemand.

/**
 * Use the IteratorBuildingVisitor limit to the current docRange to parse all delayed sub trees of the query. From those delayed sub trees initialize all
 * iterators matching the target field and aggregate all partial Documents into a list
 *
 * @param name
 *            the name of the field to fetch on demand
 * @return the list of Document objects that were fetched from all delayed iterators associated with the on-demand field
 */
private List<Document> fetchOnDemand(String name) throws IOException {
    List<Document> documentList = new ArrayList<>();
    // limit the ranges to use to the current document
    iteratorBuildingVisitor.limit(docRange);
    // for each sub tree build the nested iterator
    for (JexlNode delayedNonEventNode : delayedNonEventFieldMap.get(name)) {
        // sanity check
        if (delayedNonEventNode == null) {
            throw new IllegalStateException("Delayed nonEventNode must not be null");
        }
        // reset the root
        iteratorBuildingVisitor.resetRoot();
        // construct the index iterator for this node
        delayedNonEventNode.jjtAccept(iteratorBuildingVisitor, null);
        NestedIterator<Key> delayedNodeIterator = iteratorBuildingVisitor.root();
        if (delayedNodeIterator != null) {
            // get all the leaf nodes, this is very likely (always?)
            Collection<NestedIterator<Key>> leaves = delayedNodeIterator.leaves();
            // for each leaf, see if its a match for the target field
            for (NestedIterator<Key> leaf : leaves) {
                // init/seek the leaf
                leaf.initialize();
                if (leaf instanceof SeekableIterator) {
                    ((SeekableIterator) leaf).seek(docRange, columnFamilies, inclusive);
                }
                // for each value off the leaf add it to the document list as long as equality accepts it
                while (leaf.hasNext()) {
                    Key nextKey = leaf.next();
                    if (equality.partOf(docRange.getStartKey(), nextKey)) {
                        documentList.add(leaf.document());
                    }
                }
            }
        }
    }
    return documentList;
}

Example 9 with NestedIterator

use of datawave.query.iterator.NestedIterator in project datawave by NationalSecurityAgency.

the class AndOrIteratorTest method testAndAlwaysTrue.

// X AND (Y OR !Y)
@Test
public void testAndAlwaysTrue() {
    Set<NestedIterator<String>> childIncludes = new HashSet<>();
    Set<NestedIterator<String>> childExcludes = new HashSet<>();
    childIncludes.add(getItr(Lists.newArrayList("b", "c"), false));
    childExcludes.add(getItr(Lists.newArrayList("b", "c"), false));
    NestedIterator child = new OrIterator(childIncludes, childExcludes);
    Set<NestedIterator<String>> includes = new HashSet<>();
    includes.add(child);
    includes.add(getItr(Lists.newArrayList("a", "b", "c", "d"), false));
    NestedIterator iterator = new AndIterator(includes);
    iterator.initialize();
    Assert.assertFalse(iterator.isContextRequired());
    Assert.assertTrue(child.isContextRequired());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("a", iterator.next());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("b", iterator.next());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("c", iterator.next());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("d", iterator.next());
    Assert.assertFalse(iterator.hasNext());
}

Example 10 with NestedIterator

use of datawave.query.iterator.NestedIterator in project datawave by NationalSecurityAgency.

the class AndOrIteratorTest method testAndMixedOr.

// X AND (Y OR !Z)
@Test
public void testAndMixedOr() {
    Set<NestedIterator<String>> childIncludes = new HashSet<>();
    Set<NestedIterator<String>> childExcludes = new HashSet<>();
    childIncludes.add(getItr(Lists.newArrayList("b", "c"), false));
    childExcludes.add(getItr(Lists.newArrayList("a", "b"), false));
    OrIterator childOr = new OrIterator(childIncludes, childExcludes);
    Set<NestedIterator<String>> includes = new HashSet<>();
    includes.add(childOr);
    includes.add(getItr(Lists.newArrayList("a", "b", "c"), false));
    NestedIterator iterator = new AndIterator(includes);
    iterator.initialize();
    Assert.assertFalse(iterator.isContextRequired());
    Assert.assertTrue(childOr.isContextRequired());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("b", iterator.next());
    Assert.assertTrue(iterator.hasNext());
    Assert.assertEquals("c", iterator.next());
    Assert.assertFalse(iterator.hasNext());
}