Examples with ScannerChunk datawave.query.tables.async.ScannerChunk used on opensource projects

Example 6 with ScannerChunk

use of datawave.query.tables.async.ScannerChunk in project datawave by NationalSecurityAgency.

the class VisitorFunctionTest method rangeOverTermThresholdTest.

@Test
public void rangeOverTermThresholdTest() throws IOException, TableNotFoundException, URISyntaxException {
    setupExpects();
    config.setCleanupShardsAndDaysQueryHints(false);
    config.setBypassExecutabilityCheck(true);
    config.setSerializeQueryIterator(false);
    Query mockQuery = createMock(Query.class);
    config.setQuery(mockQuery);
    EasyMock.expect(mockQuery.getId()).andReturn(new UUID(0, 0)).anyTimes();
    EasyMock.expect(mockQuery.duplicate("testQuery1")).andReturn(mockQuery).anyTimes();
    // set thresholds
    config.setMaxTermThreshold(1);
    config.setMaxDepthThreshold(10);
    config.setMaxOrExpansionFstThreshold(100);
    config.setMaxOrExpansionThreshold(20);
    config.setMaxOrRangeThreshold(2);
    config.setMaxRangesPerRangeIvarator(50);
    config.setMaxOrRangeThreshold(2);
    SessionOptions options = new SessionOptions();
    IteratorSetting iteratorSetting = new IteratorSetting(10, "itr", QueryIterator.class);
    String query = "((_Bounded_ = true) && (FIELD1 > 'a' && FIELD1 < 'y')) || ((_Bounded_ = true) && (FIELD1 > 'c' && FIELD1 < 'z'))";
    iteratorSetting.addOption(QueryOptions.QUERY, query);
    options.addScanIterator(iteratorSetting);
    ScannerChunk chunk = new ScannerChunk(options, Collections.singleton(new Range("20210101_0", "20210101_0")));
    replayAll();
    function = new VisitorFunction(config, helper);
    ScannerChunk updatedChunk = function.apply(chunk);
    verifyAll();
    Assert.assertNotEquals(chunk, updatedChunk);
    String updatedQuery = updatedChunk.getOptions().getIterators().iterator().next().getOptions().get(QueryOptions.QUERY);
    Assert.assertNotEquals(query, updatedQuery);
    Assert.assertTrue(updatedQuery, updatedQuery.contains("_List_"));
    Assert.assertTrue(updatedQuery, updatedQuery.contains("field = 'FIELD1'"));
    Assert.assertTrue(updatedQuery, updatedQuery.contains("ranges\":[[\"(a\",\"z)\"]"));
}

Example 7 with ScannerChunk

use of datawave.query.tables.async.ScannerChunk in project datawave by NationalSecurityAgency.

the class VisitorFunctionTest method underTermThresholdTest.

@Test
public void underTermThresholdTest() throws IOException, TableNotFoundException, URISyntaxException {
    setupExpects();
    // test specific expects
    config.setCleanupShardsAndDaysQueryHints(false);
    config.setBypassExecutabilityCheck(true);
    config.setSerializeQueryIterator(false);
    Query mockQuery = createMock(Query.class);
    config.setQuery(mockQuery);
    EasyMock.expect(mockQuery.getId()).andReturn(new UUID(0, 0)).anyTimes();
    // set thresholds
    config.setMaxTermThreshold(2);
    config.setMaxDepthThreshold(2);
    SessionOptions options = new SessionOptions();
    IteratorSetting iteratorSetting = new IteratorSetting(10, "itr", QueryIterator.class);
    iteratorSetting.addOption(QueryOptions.QUERY, "FIELD1 == 'a'");
    options.addScanIterator(iteratorSetting);
    ScannerChunk chunk = new ScannerChunk(options, Collections.singleton(new Range("20210101_0", "20210101_0")));
    replayAll();
    function = new VisitorFunction(config, helper);
    function.apply(chunk);
    verifyAll();
}

Example 8 with ScannerChunk

use of datawave.query.tables.async.ScannerChunk in project datawave by NationalSecurityAgency.

the class VisitorFunctionTest method overTermThresholdCantReduceTest.

@Test(expected = DatawaveFatalQueryException.class)
public void overTermThresholdCantReduceTest() throws IOException, TableNotFoundException, URISyntaxException {
    setupExpects();
    config.setCleanupShardsAndDaysQueryHints(false);
    config.setBypassExecutabilityCheck(true);
    config.setSerializeQueryIterator(false);
    Query mockQuery = createMock(Query.class);
    config.setQuery(mockQuery);
    EasyMock.expect(mockQuery.getId()).andReturn(new UUID(0, 0)).anyTimes();
    EasyMock.expect(mockQuery.getQueryName()).andReturn("testQuery1").anyTimes();
    EasyMock.expect(mockQuery.duplicate("testQuery1")).andReturn(mockQuery).anyTimes();
    // set thresholds
    config.setMaxTermThreshold(1);
    config.setMaxDepthThreshold(10);
    config.setMaxOrExpansionFstThreshold(100);
    config.setMaxOrExpansionThreshold(20);
    config.setMaxOrRangeThreshold(2);
    config.setMaxRangesPerRangeIvarator(50);
    config.setMaxOrRangeThreshold(2);
    SessionOptions options = new SessionOptions();
    IteratorSetting iteratorSetting = new IteratorSetting(10, "itr", QueryIterator.class);
    String query = "FIELD2 == 'a' || FIELD1 == 'b'";
    iteratorSetting.addOption(QueryOptions.QUERY, query);
    options.addScanIterator(iteratorSetting);
    ScannerChunk chunk = new ScannerChunk(options, Collections.singleton(new Range("20210101_0", "20210101_0")));
    replayAll();
    function = new VisitorFunction(config, helper);
    ScannerChunk updatedChunk = function.apply(chunk);
    verifyAll();
    Assert.assertNotEquals(chunk, updatedChunk);
    String updatedQuery = updatedChunk.getOptions().getIterators().iterator().next().getOptions().get(QueryOptions.QUERY);
    Assert.assertNotEquals(query, updatedQuery);
    Assert.assertTrue(updatedQuery, updatedQuery.contains("_List_"));
    Assert.assertTrue(updatedQuery, updatedQuery.contains("field = 'FIELD1'"));
    Assert.assertTrue(updatedQuery, updatedQuery.contains("values\":[\"a\",\"b\"]"));
}

Example 9 with ScannerChunk

use of datawave.query.tables.async.ScannerChunk in project datawave by NationalSecurityAgency.

the class BatchScannerSession method submitTasks.

/**
 */
protected void submitTasks() {
    List<ScannerChunk> newChunks;
    newChunks = Lists.newArrayList(currentBatch);
    currentBatch.clear();
    Collections.shuffle(newChunks);
    for (ScannerChunk chunk : newChunks) {
        AtomicInteger numChunks = serverMap.get(chunk.getLastKnownLocation());
        if (numChunks == null) {
            numChunks = new AtomicInteger(1);
            serverMap.put(chunk.getLastKnownLocation(), numChunks);
        } else
            numChunks.incrementAndGet();
        Scan scan = null;
        if (speculativeScanning && delegatedResourceInitializer == RfileResource.class) {
            if (log.isTraceEnabled()) {
                log.trace("Using speculative execution");
            }
            scan = new SpeculativeScan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
            ((SpeculativeScan) scan).addScan(new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, BatchResource.class, ((SpeculativeScan) scan).getQueue(), listenerService));
            ((SpeculativeScan) scan).addScan(new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, delegatedResourceInitializer, ((SpeculativeScan) scan).getQueue(), listenerService));
        } else {
            scan = new Scan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
        }
        if (backoffEnabled) {
            scan.setSessionArbiter(this);
        }
        scan.setVisitors(visitorFunctions);
        scan.setTimeout(scanLimitTimeout);
        if (log.isTraceEnabled()) {
            log.trace("Adding scanner " + chunk);
        }
        submitScan(scan, true);
    }
}