use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.
the class ContentQueryTable method close.
/**
* This method calls the base logic's close method, and then attempts to close all batch scanners tracked by the scanner factory, if it is not null.
*/
@Override
public void close() {
super.close();
final ScannerFactory factory = this.scannerFactory;
if (null == factory) {
log.debug("ScannerFactory is null; not closing it.");
} else {
int nClosed = 0;
factory.lockdown();
for (final ScannerBase bs : Lists.newArrayList(factory.currentScanners())) {
factory.close(bs);
++nClosed;
}
if (log.isDebugEnabled())
log.debug("Cleaned up " + nClosed + " batch scanners associated with this query logic.");
}
}
use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.
the class EdgeQueryLogic method initialize.
@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
currentIteratorPriority = super.getBaseIteratorPriority() + 30;
EdgeQueryConfiguration cfg = setUpConfig(settings);
cfg.setConnector(connection);
cfg.setAuthorizations(auths);
String queryString = settings.getQuery();
if (null == queryString) {
throw new IllegalArgumentException("Query cannot be null");
} else {
cfg.setQueryString(queryString);
}
cfg.setBeginDate(settings.getBeginDate());
cfg.setEndDate(settings.getEndDate());
scannerFactory = new ScannerFactory(connection);
return cfg;
}
use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.
the class DefaultQueryPlanner method initializeRangeStream.
/**
* Initializes the range stream, whether it is configured to be a different class than the Default Range stream or not.
*
* @param config
* @param scannerFactory
* @param metadataHelper
* @return
*/
private RangeStream initializeRangeStream(ShardQueryConfiguration config, ScannerFactory scannerFactory, MetadataHelper metadataHelper) {
Class<? extends RangeStream> rstream;
try {
rstream = Class.forName(rangeStreamClass).asSubclass(RangeStream.class);
RangeStream stream = rstream.getConstructor(ShardQueryConfiguration.class, ScannerFactory.class, MetadataHelper.class).newInstance(config, scannerFactory, metadataHelper);
return stream.setUidIntersector(uidIntersector).setLimitScanners(limitScanners).setCreateUidsIteratorClass(createUidsIteratorClass);
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
throw new RuntimeException(e);
}
}
use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testBothIndexedPrune.
@Test
public void testBothIndexedPrune() throws Exception {
String originalQuery = "(FOO == 'barter' || FOO == 'baggy')";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
config.setBeginDate(new Date(0));
config.setEndDate(new Date(System.currentTimeMillis()));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Range range1 = makeTestRange("20190314_1", "datatype1\u0000123");
Range range2 = makeTestRange("20190314_1", "datatype1\u0000345");
Range range3 = makeTestRange("20190414_1", "datatype1\u0000123");
Range range4 = makeTestRange("20190414_1", "datatype1\u0000345");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
for (Range range : queryPlan.getRanges()) {
assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.
the class RangeStreamTest method testIntersection_NestedUnionOfLowCardinalityTerm_withSeek.
// A && (B || C)
@Test
public void testIntersection_NestedUnionOfLowCardinalityTerm_withSeek() throws Exception {
String originalQuery = "(FOO == 'highest_card' && (FOO == 'low_card' || FOO == 'lowest_card'))";
ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
config.setBeginDate(sdf.parse("20190310"));
config.setEndDate(sdf.parse("20190320"));
config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
Multimap<String, Type<?>> dataTypes = HashMultimap.create();
dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
config.setQueryFieldsDatatypes(dataTypes);
config.setIndexedFields(dataTypes);
MockMetadataHelper helper = new MockMetadataHelper();
helper.setIndexedFields(dataTypes.keySet());
Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
Range range2 = makeTestRange("20190312_1", "datatype1\u0000a.b.c");
Range range3 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
Range range4 = makeTestRange("20190315_33", "datatype1\u0000a.b.c");
Range range5 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
Range range6 = makeTestRange("20190317_1", "datatype1\u0000a.b.c");
Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
rangeStream.setLimitScanners(true);
CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
for (QueryPlan queryPlan : queryPlans) {
Iterable<Range> ranges = queryPlan.getRanges();
for (Range range : ranges) {
assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
}
}
assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Aggregations