Examples with IndexIteratorBridge datawave.query.iterator.logic.IndexIteratorBridge used on opensource projects

Example 1 with IndexIteratorBridge

use of datawave.query.iterator.logic.IndexIteratorBridge in project datawave by NationalSecurityAgency.

the class IndexFilterIteratorBuilder method build.

@SuppressWarnings("unchecked")
@Override
public NestedIterator<Key> build() {
    if (notNull(range, filter, source, datatypeFilter, keyTform, timeFilter, ivaratorCacheDirs, getField(), getNode())) {
        if (log.isTraceEnabled()) {
            log.trace("Generating ivarator (caching field index iterator) for " + filter + " over " + range);
        }
        // we can't build an ivarator if no ivarator directories have been defined
        if (ivaratorCacheDirs.isEmpty())
            throw new IllegalStateException("No ivarator cache dirs defined");
        // ensure that we are able to create the first ivarator cache dir (the control dir)
        validateIvaratorControlDir(ivaratorCacheDirs.get(0));
        DocumentIterator docIterator = null;
        try {
            // create a field index caching ivarator
            // @formatter:off
            DatawaveFieldIndexFilterIteratorJexl rangeIterator = DatawaveFieldIndexFilterIteratorJexl.builder().withFieldName(new Text(range.getFieldName())).withFilter(filter).withLowerBound(range.getLower().toString()).lowerInclusive(range.isLowerInclusive()).withUpperBound(range.getUpper().toString()).upperInclusive(range.isUpperInclusive()).withTimeFilter(timeFilter).withDatatypeFilter(datatypeFilter).negated(false).withScanThreshold(ivaratorCacheScanPersistThreshold).withScanTimeout(ivaratorCacheScanTimeout).withHdfsBackedSetBufferSize(ivaratorCacheBufferSize).withMaxRangeSplit(maxRangeSplit).withMaxOpenFiles(ivaratorMaxOpenFiles).withMaxResults(maxIvaratorResults).withIvaratorCacheDirs(ivaratorCacheDirs).withNumRetries(ivaratorNumRetries).withPersistOptions(ivaratorPersistOptions).withQueryLock(queryLock).allowDirResuse(true).withReturnKeyType(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME).withSortedUUIDs(sortedUIDs).withCompositeMetadata(compositeMetadata).withCompositeSeekThreshold(compositeSeekThreshold).withTypeMetadata(typeMetadata).withIteratorEnv(env).withIvaratorSourcePool(ivaratorSourcePool).build();
            if (collectTimingDetails) {
                rangeIterator.setCollectTimingDetails(true);
                rangeIterator.setQuerySpanCollector(this.querySpanCollector);
            }
            rangeIterator.init(source, null, env);
            log.debug("Created a DatawaveFieldIndexFilterIteratorJexl: " + rangeIterator);
            // Add an interator to aggregate documents. This is needed for index only fields.
            DocumentAggregatingIterator aggregatingIterator = new // this.fieldsToKeep == null ? false
            DocumentAggregatingIterator(// this.fieldsToKeep == null ? false
            true, // :this.fieldsToKeep.contains(field),
            this.typeMetadata, keyTform);
            aggregatingIterator.init(rangeIterator, null, null);
            docIterator = aggregatingIterator;
        } catch (IOException e) {
            throw new IllegalStateException("Unable to initialize regex iterator stack", e);
        }
        IndexIteratorBridge itr = new IndexIteratorBridge(docIterator, getNode(), getField());
        range = null;
        filter = null;
        source = null;
        timeFilter = null;
        datatypeFilter = null;
        keyTform = null;
        timeFilter = null;
        ivaratorCacheDirs = null;
        node = null;
        field = null;
        return itr;
    } else {
        StringBuilder msg = new StringBuilder(256);
        msg.append("Cannot build iterator-- a field was null!\n");
        if (filter == null) {
            msg.append("\tFilter was null!\n");
        }
        if (range == null) {
            msg.append("\tRange was null!\n");
        }
        if (source == null) {
            msg.append("\tSource was null!\n");
        }
        msg.setLength(msg.length() - 1);
        throw new IllegalStateException(msg.toString());
    }
}

Example 2 with IndexIteratorBridge

use of datawave.query.iterator.logic.IndexIteratorBridge in project datawave by NationalSecurityAgency.

the class IndexListIteratorBuilder method build.

@SuppressWarnings("unchecked")
@Override
public NestedIterator<Key> build() {
    if (notNull(field, (values != null ? values : fst), negated, source, datatypeFilter, timeFilter, keyTform, ivaratorCacheDirs, getField(), getNode())) {
        if (log.isTraceEnabled()) {
            log.trace("Generating ivarator (caching field index iterator) for " + field + (negated ? "!~" : "=~") + value);
        }
        // we can't build an ivarator if no ivarator directories have been defined
        if (ivaratorCacheDirs.isEmpty())
            throw new IllegalStateException("No ivarator cache dirs defined");
        // ensure that we are able to create the first ivarator cache dir (the control dir)
        validateIvaratorControlDir(ivaratorCacheDirs.get(0));
        DocumentIterator docIterator = null;
        try {
            // create a field index caching ivarator
            // @formatter:off
            DatawaveFieldIndexListIteratorJexl.Builder builder = DatawaveFieldIndexListIteratorJexl.builder().withFieldName(new Text(field)).withTimeFilter(timeFilter).withDatatypeFilter(datatypeFilter).negated(negated).withScanThreshold(ivaratorCacheScanPersistThreshold).withScanTimeout(ivaratorCacheScanTimeout).withHdfsBackedSetBufferSize(ivaratorCacheBufferSize).withMaxRangeSplit(maxRangeSplit).withMaxOpenFiles(ivaratorMaxOpenFiles).withIvaratorCacheDirs(ivaratorCacheDirs).withNumRetries(ivaratorNumRetries).withPersistOptions(ivaratorPersistOptions).withMaxResults(maxIvaratorResults).withQueryLock(queryLock).allowDirResuse(true).withReturnKeyType(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME).withSortedUUIDs(sortedUIDs).withCompositeMetadata(compositeMetadata).withCompositeSeekThreshold(compositeSeekThreshold).withTypeMetadata(typeMetadata).withIvaratorSourcePool(ivaratorSourcePool).withIteratorEnv(env);
            // @formatter:on
            if (values != null) {
                builder = builder.withValues(values);
            } else {
                builder = builder.withFST(fst);
            }
            DatawaveFieldIndexListIteratorJexl listIterator = builder.build();
            if (collectTimingDetails) {
                listIterator.setCollectTimingDetails(true);
                listIterator.setQuerySpanCollector(this.querySpanCollector);
            }
            listIterator.init(source, null, env);
            log.debug("Created a DatawaveFieldIndexListIteratorJexl: " + listIterator);
            boolean canBuildDocument = this.fieldsToAggregate == null ? false : this.fieldsToAggregate.contains(field);
            if (forceDocumentBuild) {
                canBuildDocument = true;
            }
            // Add an interator to aggregate documents. This is needed for index only fields.
            DocumentAggregatingIterator aggregatingIterator = new DocumentAggregatingIterator(canBuildDocument, this.typeMetadata, keyTform);
            aggregatingIterator.init(listIterator, null, null);
            docIterator = aggregatingIterator;
        } catch (IOException e) {
            throw new IllegalStateException("Unable to initialize list iterator stack", e);
        }
        IndexIteratorBridge itr = new IndexIteratorBridge(docIterator, getNode(), getField());
        field = null;
        value = null;
        negated = null;
        source = null;
        timeFilter = null;
        datatypeFilter = null;
        keyTform = null;
        timeFilter = null;
        ivaratorCacheDirs = null;
        node = null;
        field = null;
        return itr;
    } else {
        StringBuilder msg = new StringBuilder(256);
        msg.append("Cannot build iterator-- a field was null!\n");
        if (field == null) {
            msg.append("\tField was null!\n");
        }
        if (value == null) {
            msg.append("\tValue was null!\n");
        }
        if (source == null) {
            msg.append("\tSource was null!\n");
        }
        msg.setLength(msg.length() - 1);
        throw new IllegalStateException(msg.toString());
    }
}

Example 3 with IndexIteratorBridge

use of datawave.query.iterator.logic.IndexIteratorBridge in project datawave by NationalSecurityAgency.

the class IndexRegexIteratorBuilder method build.

@SuppressWarnings("unchecked")
@Override
public NestedIterator<Key> build() {
    if (notNull(field, value, negated, source, datatypeFilter, timeFilter, keyTform, ivaratorCacheDirs, getField(), getNode())) {
        if (log.isTraceEnabled()) {
            log.trace("Generating ivarator (caching field index iterator) for " + field + (negated ? "!~" : "=~") + value);
        }
        // we can't build an ivarator if no ivarator directories have been defined
        if (ivaratorCacheDirs.isEmpty())
            throw new IllegalStateException("No ivarator cache dirs defined");
        // ensure that we are able to create the first ivarator cache dir (the control dir)
        validateIvaratorControlDir(ivaratorCacheDirs.get(0));
        DocumentIterator docIterator = null;
        try {
            // create a field index caching ivarator
            // @formatter:off
            DatawaveFieldIndexRegexIteratorJexl regexIterator = DatawaveFieldIndexRegexIteratorJexl.builder().withFieldName(new Text(field)).withFieldValue(new Text(value)).withTimeFilter(timeFilter).withDatatypeFilter(datatypeFilter).negated(negated).withScanThreshold(ivaratorCacheScanPersistThreshold).withScanTimeout(ivaratorCacheScanTimeout).withHdfsBackedSetBufferSize(ivaratorCacheBufferSize).withMaxRangeSplit(maxRangeSplit).withMaxOpenFiles(ivaratorMaxOpenFiles).withMaxResults(maxIvaratorResults).withIvaratorCacheDirs(ivaratorCacheDirs).withNumRetries(ivaratorNumRetries).withPersistOptions(ivaratorPersistOptions).withQueryLock(queryLock).allowDirResuse(true).withReturnKeyType(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME).withSortedUUIDs(sortedUIDs).withCompositeMetadata(compositeMetadata).withCompositeSeekThreshold(compositeSeekThreshold).withTypeMetadata(typeMetadata).withIteratorEnv(env).withIvaratorSourcePool(ivaratorSourcePool).build();
            // @formatter:on
            if (collectTimingDetails) {
                regexIterator.setCollectTimingDetails(true);
                regexIterator.setQuerySpanCollector(this.querySpanCollector);
            }
            regexIterator.init(source, null, env);
            log.debug("Created a DatawaveFieldIndexRegexIteratorJexl: " + regexIterator);
            boolean canBuildDocument = this.fieldsToAggregate == null ? false : this.fieldsToAggregate.contains(field);
            if (forceDocumentBuild) {
                canBuildDocument = true;
            }
            // Add an interator to aggregate documents. This is needed for index only fields.
            DocumentAggregatingIterator aggregatingIterator = new DocumentAggregatingIterator(canBuildDocument, this.typeMetadata, keyTform);
            aggregatingIterator.init(regexIterator, null, null);
            docIterator = aggregatingIterator;
        } catch (IOException e) {
            throw new IllegalStateException("Unable to initialize regex iterator stack", e);
        // } catch (JavaRegexParseException e) {
        // throw new IllegalStateException("Unable to parse regex " + value, e);
        }
        IndexIteratorBridge itr = new IndexIteratorBridge(docIterator, getNode(), getField());
        field = null;
        value = null;
        negated = null;
        source = null;
        timeFilter = null;
        datatypeFilter = null;
        keyTform = null;
        timeFilter = null;
        ivaratorCacheDirs = null;
        field = null;
        node = null;
        return itr;
    } else {
        StringBuilder msg = new StringBuilder(256);
        msg.append("Cannot build iterator-- a field was null!\n");
        if (field == null) {
            msg.append("\tField was null!\n");
        }
        if (value == null) {
            msg.append("\tValue was null!\n");
        }
        if (source == null) {
            msg.append("\tSource was null!\n");
        }
        msg.setLength(msg.length() - 1);
        throw new IllegalStateException(msg.toString());
    }
}

Example 4 with IndexIteratorBridge

use of datawave.query.iterator.logic.IndexIteratorBridge in project datawave by NationalSecurityAgency.

the class TermFrequencyIndexBuilder method build.

@SuppressWarnings("unchecked")
public NestedIterator<Key> build() {
    if (notNull(field, range, source, datatypeFilter, timeFilter)) {
        IndexIteratorBridge itr = new IndexIteratorBridge(new TermFrequencyIndexIterator(range, source, this.timeFilter, this.typeMetadata, this.fieldsToAggregate == null ? false : this.fieldsToAggregate.contains(field), this.datatypeFilter, termFrequencyAggregator), getNode(), getField());
        field = null;
        range = null;
        source = null;
        timeFilter = null;
        datatypeFilter = null;
        node = null;
        return itr;
    } else {
        StringBuilder msg = new StringBuilder(256);
        msg.append("Cannot build iterator-- a field was null!\n");
        if (field == null) {
            msg.append("\tField was null!\n");
        }
        if (range == null) {
            msg.append("\tValue was null!\n");
        }
        if (source == null) {
            msg.append("\tSource was null!\n");
        }
        msg.setLength(msg.length() - 1);
        throw new IllegalStateException(msg.toString());
    }
}

Example 5 with IndexIteratorBridge

use of datawave.query.iterator.logic.IndexIteratorBridge in project datawave by NationalSecurityAgency.

the class DelayedNonEventSubTreeVisitor method extractDelayedFields.

private void extractDelayedFields(JexlNode node) {
    // reset the root before processing a a node
    iteratorBuildingVisitor.resetRoot();
    // use the iterator building visitor to build a field to node map for each delayed node
    node.jjtAccept(iteratorBuildingVisitor, null);
    NestedIterator<Key> root = iteratorBuildingVisitor.root();
    if (root != null) {
        for (NestedIterator<Key> leaf : root.leaves()) {
            // only IndexIteratorBridge nodes matter, everything else can be ignored
            if (leaf instanceof IndexIteratorBridge) {
                String fieldName = ((IndexIteratorBridge) leaf).getField();
                if (nonEventFields.contains(fieldName)) {
                    JexlNode leafNode = ((IndexIteratorBridge) leaf).getSourceNode();
                    JexlNode targetNode = leafNode;
                    delayedNonEventFieldMapNodes.put(fieldName, targetNode);
                }
            }
        }
    }
}