Examples with ScannerSession datawave.query.tables.ScannerSession used on opensource projects

Example 1 with ScannerSession

use of datawave.query.tables.ScannerSession in project datawave by NationalSecurityAgency.

the class FieldNameIndexLookup method submit.

@Override
public void submit() {
    if (indexLookupMap == null) {
        indexLookupMap = new IndexLookupMap(config.getMaxUnfieldedExpansionThreshold(), config.getMaxValueExpansionThreshold());
        Iterator<Entry<Key, Value>> iter = Iterators.emptyIterator();
        ScannerSession bs;
        try {
            if (!fields.isEmpty()) {
                for (String term : terms) {
                    Set<Range> ranges = Collections.singleton(ShardIndexQueryTableStaticMethods.getLiteralRange(term));
                    if (config.getLimitAnyFieldLookups()) {
                        log.trace("Creating configureTermMatchOnly");
                        bs = ShardIndexQueryTableStaticMethods.configureTermMatchOnly(config, scannerFactory, config.getIndexTableName(), ranges, Collections.singleton(term), Collections.emptySet(), false, true);
                    } else {
                        log.trace("Creating configureLimitedDiscovery");
                        bs = ShardIndexQueryTableStaticMethods.configureLimitedDiscovery(config, scannerFactory, config.getIndexTableName(), ranges, Collections.singleton(term), Collections.emptySet(), false, true);
                    }
                    // Fetch the limited field names for the given rows
                    for (String field : fields) {
                        bs.getOptions().fetchColumnFamily(new Text(field));
                    }
                    sessions.add(bs);
                    iter = Iterators.concat(iter, bs);
                }
            }
            timedScanFuture = execService.submit(createTimedCallable(iter));
        } catch (TableNotFoundException e) {
            log.error(e);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

Example 2 with ScannerSession

use of datawave.query.tables.ScannerSession in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTableStaticMethods method configureTermMatchOnly.

/**
 * We only need to concern ourselves with looking for field names.
 *
 * @param config
 * @param scannerFactory
 * @param tableName
 * @param ranges
 * @param literals
 * @param patterns
 * @param reverseIndex
 * @param limitToUniqueTerms
 * @return
 * @throws Exception
 */
public static ScannerSession configureTermMatchOnly(ShardQueryConfiguration config, ScannerFactory scannerFactory, String tableName, Collection<Range> ranges, Collection<String> literals, Collection<String> patterns, boolean reverseIndex, boolean limitToUniqueTerms) throws Exception {
    // if we have no ranges, then nothing to scan
    if (ranges.isEmpty()) {
        return null;
    }
    ScannerSession bs = scannerFactory.newLimitedScanner(AnyFieldScanner.class, tableName, config.getAuthorizations(), config.getQuery());
    bs.setRanges(ranges);
    SessionOptions options = new SessionOptions();
    IteratorSetting setting = configureDateRangeIterator(config);
    options.addScanIterator(setting);
    setting = configureGlobalIndexTermMatchingIterator(config, literals, patterns, reverseIndex, limitToUniqueTerms);
    if (setting != null) {
        options.addScanIterator(setting);
    }
    bs.setOptions(options);
    return bs;
}

Example 3 with ScannerSession

use of datawave.query.tables.ScannerSession in project datawave by NationalSecurityAgency.

the class RegexIndexLookup method submit.

@Override
public synchronized void submit() {
    if (indexLookupMap == null) {
        indexLookupMap = new IndexLookupMap(config.getMaxUnfieldedExpansionThreshold(), config.getMaxValueExpansionThreshold());
        indexLookupMap.setPatterns(patterns);
        Multimap<String, Range> forwardMap = ArrayListMultimap.create(), reverseMap = ArrayListMultimap.create();
        // Loop over all the patterns, classifying them as forward or reverse index satisfiable
        Iterator<Entry<Key, Value>> iter = Iterators.emptyIterator();
        ScannerSession bs;
        IteratorSetting fairnessIterator = null;
        if (config.getMaxIndexScanTimeMillis() > 0) {
            // The fairness iterator solves the problem whereby we have runaway iterators as a result of an evaluation that never finds anything
            fairnessIterator = new IteratorSetting(1, TimeoutIterator.class);
            long maxTime = (long) (config.getMaxIndexScanTimeMillis() * 1.25);
            fairnessIterator.addOption(TimeoutIterator.MAX_SESSION_TIME, Long.valueOf(maxTime).toString());
        }
        for (String pattern : patterns) {
            if (config.getDisallowedRegexPatterns().contains(pattern)) {
                PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.IGNORE_PATTERN_FOR_INDEX_LOOKUP, MessageFormat.format("Pattern: {0}", pattern));
                log.debug(qe);
                throw new DoNotPerformOptimizedQueryException(qe);
            }
            ShardIndexQueryTableStaticMethods.RefactoredRangeDescription rangeDescription;
            try {
                rangeDescription = ShardIndexQueryTableStaticMethods.getRegexRange(null, pattern, config.getFullTableScanEnabled(), helper, config);
            } catch (IllegalArgumentException | JavaRegexParseException e) {
                log.debug("Ignoring pattern that was not capable of being looked up in the index: " + pattern, e);
                continue;
            } catch (TableNotFoundException e) {
                log.error(e);
                throw new DatawaveFatalQueryException(e);
            } catch (ExecutionException e) {
                throw new DatawaveFatalQueryException(e);
            }
            if (log.isTraceEnabled()) {
                log.trace("Adding pattern " + pattern);
                log.trace("Adding pattern " + rangeDescription);
            }
            if (rangeDescription.isForReverseIndex) {
                reverseMap.put(pattern, rangeDescription.range);
            } else {
                forwardMap.put(pattern, rangeDescription.range);
            }
        }
        if (!fields.isEmpty() && !forwardMap.isEmpty()) {
            for (String key : forwardMap.keySet()) {
                Collection<Range> ranges = forwardMap.get(key);
                try {
                    bs = ShardIndexQueryTableStaticMethods.configureLimitedDiscovery(config, scannerFactory, config.getIndexTableName(), ranges, Collections.emptySet(), Collections.singleton(key), false, true);
                    bs.setResourceClass(BatchResource.class);
                } catch (Exception e) {
                    throw new DatawaveFatalQueryException(e);
                }
                SessionOptions opts = bs.getOptions();
                if (null != fairnessIterator) {
                    opts.addScanIterator(fairnessIterator);
                    IteratorSetting cfg = new IteratorSetting(config.getBaseIteratorPriority() + 100, TimeoutExceptionIterator.class);
                    opts.addScanIterator(cfg);
                }
                for (String field : fields) {
                    opts.fetchColumnFamily(new Text(field));
                }
                forwardLookupData.getSessions().add(bs);
                iter = Iterators.concat(iter, bs);
            }
            forwardLookupData.setTimedScanFuture(execService.submit(createTimedCallable(iter, fields, forwardLookupData, indexLookupMap)));
        }
        if (!reverseFields.isEmpty() && !reverseMap.isEmpty()) {
            for (String key : reverseMap.keySet()) {
                Collection<Range> ranges = reverseMap.get(key);
                if (log.isTraceEnabled()) {
                    log.trace("adding " + ranges + " for reverse");
                }
                try {
                    bs = ShardIndexQueryTableStaticMethods.configureLimitedDiscovery(config, scannerFactory, config.getReverseIndexTableName(), ranges, Collections.emptySet(), Collections.singleton(key), true, true);
                    bs.setResourceClass(BatchResource.class);
                } catch (Exception e) {
                    throw new DatawaveFatalQueryException(e);
                }
                SessionOptions opts = bs.getOptions();
                if (null != fairnessIterator) {
                    opts.addScanIterator(fairnessIterator);
                    opts.addScanIterator(new IteratorSetting(config.getBaseIteratorPriority() + 100, TimeoutExceptionIterator.class));
                }
                for (String field : reverseFields) {
                    opts.fetchColumnFamily(new Text(field));
                }
                reverseLookupData.getSessions().add(bs);
                iter = Iterators.concat(iter, bs);
            }
            reverseLookupData.setTimedScanFuture(execService.submit(createTimedCallable(iter, reverseFields, reverseLookupData, indexLookupMap)));
        }
    }
}

Example 4 with ScannerSession

use of datawave.query.tables.ScannerSession in project datawave by NationalSecurityAgency.

the class ShardIndexQueryTableStaticMethods method configureLimitedDiscovery.

public static ScannerSession configureLimitedDiscovery(ShardQueryConfiguration config, ScannerFactory scannerFactory, String tableName, Collection<Range> ranges, Collection<String> literals, Collection<String> patterns, boolean reverseIndex, boolean limitToUniqueTerms) throws Exception {
    // if we have no ranges, then nothing to scan
    if (ranges.isEmpty()) {
        return null;
    }
    ScannerSession bs = scannerFactory.newLimitedScanner(AnyFieldScanner.class, tableName, config.getAuthorizations(), config.getQuery());
    bs.setRanges(ranges);
    SessionOptions options = new SessionOptions();
    options.addScanIterator(configureDateRangeIterator(config));
    IteratorSetting setting = configureGlobalIndexDataTypeFilter(config, config.getDatatypeFilter());
    if (setting != null) {
        options.addScanIterator(setting);
    }
    setting = configureGlobalIndexTermMatchingIterator(config, literals, patterns, reverseIndex, limitToUniqueTerms);
    if (setting != null) {
        options.addScanIterator(setting);
    }
    bs.setOptions(options);
    return bs;
}