Search in sources :

Example 1 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class ContentQueryTable method close.

/**
 * This method calls the base logic's close method, and then attempts to close all batch scanners tracked by the scanner factory, if it is not null.
 */
@Override
public void close() {
    super.close();
    final ScannerFactory factory = this.scannerFactory;
    if (null == factory) {
        log.debug("ScannerFactory is null; not closing it.");
    } else {
        int nClosed = 0;
        factory.lockdown();
        for (final ScannerBase bs : Lists.newArrayList(factory.currentScanners())) {
            factory.close(bs);
            ++nClosed;
        }
        if (log.isDebugEnabled())
            log.debug("Cleaned up " + nClosed + " batch scanners associated with this query logic.");
    }
}
Also used : ScannerBase(org.apache.accumulo.core.client.ScannerBase) ScannerFactory(datawave.query.tables.ScannerFactory)

Example 2 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class EdgeQueryLogic method initialize.

@Override
public GenericQueryConfiguration initialize(Connector connection, Query settings, Set<Authorizations> auths) throws Exception {
    currentIteratorPriority = super.getBaseIteratorPriority() + 30;
    EdgeQueryConfiguration cfg = setUpConfig(settings);
    cfg.setConnector(connection);
    cfg.setAuthorizations(auths);
    String queryString = settings.getQuery();
    if (null == queryString) {
        throw new IllegalArgumentException("Query cannot be null");
    } else {
        cfg.setQueryString(queryString);
    }
    cfg.setBeginDate(settings.getBeginDate());
    cfg.setEndDate(settings.getEndDate());
    scannerFactory = new ScannerFactory(connection);
    return cfg;
}
Also used : ScannerFactory(datawave.query.tables.ScannerFactory) EdgeQueryConfiguration(datawave.query.config.EdgeQueryConfiguration)

Example 3 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class DefaultQueryPlanner method initializeRangeStream.

/**
 * Initializes the range stream, whether it is configured to be a different class than the Default Range stream or not.
 *
 * @param config
 * @param scannerFactory
 * @param metadataHelper
 * @return
 */
private RangeStream initializeRangeStream(ShardQueryConfiguration config, ScannerFactory scannerFactory, MetadataHelper metadataHelper) {
    Class<? extends RangeStream> rstream;
    try {
        rstream = Class.forName(rangeStreamClass).asSubclass(RangeStream.class);
        RangeStream stream = rstream.getConstructor(ShardQueryConfiguration.class, ScannerFactory.class, MetadataHelper.class).newInstance(config, scannerFactory, metadataHelper);
        return stream.setUidIntersector(uidIntersector).setLimitScanners(limitScanners).setCreateUidsIteratorClass(createUidsIteratorClass);
    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
        throw new RuntimeException(e);
    }
}
Also used : AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) ScannerFactory(datawave.query.tables.ScannerFactory) InvocationTargetException(java.lang.reflect.InvocationTargetException) MetadataHelper(datawave.query.util.MetadataHelper) RangeStream(datawave.query.index.lookup.RangeStream) ShardQueryConfiguration(datawave.query.config.ShardQueryConfiguration)

Example 4 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testBothIndexedPrune.

@Test
public void testBothIndexedPrune() throws Exception {
    String originalQuery = "(FOO == 'barter' || FOO == 'baggy')";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    config.setBeginDate(new Date(0));
    config.setEndDate(new Date(System.currentTimeMillis()));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("NUM", Sets.newHashSet(new NumberType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190314_1", "datatype1\u0000123");
    Range range2 = makeTestRange("20190314_1", "datatype1\u0000345");
    Range range3 = makeTestRange("20190414_1", "datatype1\u0000123");
    Range range4 = makeTestRange("20190414_1", "datatype1\u0000345");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper).setLimitScanners(true);
    for (QueryPlan queryPlan : rangeStream.streamPlans(script)) {
        for (Range range : queryPlan.getRanges()) {
            assertTrue("Tried to remove unexpected range " + range.toString() + " from expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) NumberType(datawave.data.type.NumberType) Test(org.junit.Test)

Example 5 with ScannerFactory

use of datawave.query.tables.ScannerFactory in project datawave by NationalSecurityAgency.

the class RangeStreamTest method testIntersection_NestedUnionOfLowCardinalityTerm_withSeek.

// A && (B || C)
@Test
public void testIntersection_NestedUnionOfLowCardinalityTerm_withSeek() throws Exception {
    String originalQuery = "(FOO == 'highest_card' && (FOO == 'low_card' || FOO == 'lowest_card'))";
    ASTJexlScript script = JexlASTHelper.parseJexlQuery(originalQuery);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    config.setBeginDate(sdf.parse("20190310"));
    config.setEndDate(sdf.parse("20190320"));
    config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2"));
    Multimap<String, Type<?>> dataTypes = HashMultimap.create();
    dataTypes.putAll("FOO", Sets.newHashSet(new LcNoDiacriticsType()));
    dataTypes.putAll("LAUGH", Sets.newHashSet(new LcNoDiacriticsType()));
    config.setQueryFieldsDatatypes(dataTypes);
    config.setIndexedFields(dataTypes);
    MockMetadataHelper helper = new MockMetadataHelper();
    helper.setIndexedFields(dataTypes.keySet());
    Range range1 = makeTestRange("20190310_1", "datatype1\u0000a.b.c");
    Range range2 = makeTestRange("20190312_1", "datatype1\u0000a.b.c");
    Range range3 = makeTestRange("20190314_22", "datatype1\u0000a.b.c");
    Range range4 = makeTestRange("20190315_33", "datatype1\u0000a.b.c");
    Range range5 = makeTestRange("20190315_49", "datatype1\u0000a.b.c");
    Range range6 = makeTestRange("20190317_1", "datatype1\u0000a.b.c");
    Set<Range> expectedRanges = Sets.newHashSet(range1, range2, range3, range4, range5, range6);
    RangeStream rangeStream = new RangeStream(config, new ScannerFactory(config.getConnector(), 1), helper);
    rangeStream.setLimitScanners(true);
    CloseableIterable<QueryPlan> queryPlans = rangeStream.streamPlans(script);
    assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context());
    for (QueryPlan queryPlan : queryPlans) {
        Iterable<Range> ranges = queryPlan.getRanges();
        for (Range range : ranges) {
            assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges.toString(), expectedRanges.remove(range));
        }
    }
    assertTrue("Expected ranges not found in query plan: " + expectedRanges.toString(), expectedRanges.isEmpty());
}
Also used : ASTJexlScript(org.apache.commons.jexl2.parser.ASTJexlScript) RangeFactoryForTests.makeTestRange(datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange) Range(org.apache.accumulo.core.data.Range) ScannerFactory(datawave.query.tables.ScannerFactory) QueryPlan(datawave.query.planner.QueryPlan) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) MockMetadataHelper(datawave.query.util.MockMetadataHelper) NoOpType(datawave.data.type.NoOpType) NumberType(datawave.data.type.NumberType) LcNoDiacriticsType(datawave.data.type.LcNoDiacriticsType) Type(datawave.data.type.Type) SimpleDateFormat(java.text.SimpleDateFormat) Test(org.junit.Test)

Aggregations

ScannerFactory (datawave.query.tables.ScannerFactory)40 Type (datawave.data.type.Type)36 ASTJexlScript (org.apache.commons.jexl2.parser.ASTJexlScript)35 LcNoDiacriticsType (datawave.data.type.LcNoDiacriticsType)34 MockMetadataHelper (datawave.query.util.MockMetadataHelper)34 NoOpType (datawave.data.type.NoOpType)33 NumberType (datawave.data.type.NumberType)33 Test (org.junit.Test)32 Range (org.apache.accumulo.core.data.Range)26 RangeFactoryForTests.makeTestRange (datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange)25 QueryPlan (datawave.query.planner.QueryPlan)25 SimpleDateFormat (java.text.SimpleDateFormat)9 ShardQueryConfiguration (datawave.query.config.ShardQueryConfiguration)2 MetadataHelper (datawave.query.util.MetadataHelper)2 Stopwatch (com.google.common.base.Stopwatch)1 ContentQueryConfiguration (datawave.query.config.ContentQueryConfiguration)1 EdgeQueryConfiguration (datawave.query.config.EdgeQueryConfiguration)1 QueryValues (datawave.query.discovery.FindLiteralsAndPatternsVisitor.QueryValues)1 RangeStream (datawave.query.index.lookup.RangeStream)1 LuceneToJexlQueryParser (datawave.query.language.parser.jexl.LuceneToJexlQueryParser)1