Search in sources :

Example 1 with SpeculativeScan

use of datawave.query.tables.async.SpeculativeScan in project datawave by NationalSecurityAgency.

the class BatchScannerSession method submitTasks.

protected void submitTasks(List<ScannerChunk> newChunks) {
    for (ScannerChunk chunk : newChunks) {
        AtomicInteger numChunks = serverMap.get(chunk.getLastKnownLocation());
        if (numChunks == null) {
            numChunks = new AtomicInteger(1);
            serverMap.put(chunk.getLastKnownLocation(), numChunks);
        } else
            numChunks.incrementAndGet();
        Scan scan = null;
        if (speculativeScanning && delegatedResourceInitializer == RfileResource.class) {
            if (log.isTraceEnabled()) {
                log.trace("Using speculative execution");
            }
            chunk.setQueryId(settings.getId().toString());
            scan = new SpeculativeScan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
            scan.setVisitors(visitorFunctions);
            Scan childScan = new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, BatchResource.class, ((SpeculativeScan) scan).getQueue(), listenerService);
            childScan.setVisitors(visitorFunctions);
            ((SpeculativeScan) scan).addScan(childScan);
            childScan = new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, delegatedResourceInitializer, ((SpeculativeScan) scan).getQueue(), listenerService);
            childScan.setVisitors(visitorFunctions);
            ((SpeculativeScan) scan).addScan(childScan);
        } else {
            scan = new Scan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
        }
        if (backoffEnabled) {
            scan.setSessionArbiter(this);
        }
        scan.setVisitors(visitorFunctions);
        scan.setTimeout(scanLimitTimeout);
        if (log.isTraceEnabled()) {
            log.trace("Adding scanner " + chunk);
        }
        submitScan(scan, true);
    }
}
Also used : RfileResource(datawave.mr.bulk.RfileResource) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SpeculativeScan(datawave.query.tables.async.SpeculativeScan) ScannerChunk(datawave.query.tables.async.ScannerChunk) Scan(datawave.query.tables.async.Scan) SpeculativeScan(datawave.query.tables.async.SpeculativeScan)

Example 2 with SpeculativeScan

use of datawave.query.tables.async.SpeculativeScan in project datawave by NationalSecurityAgency.

the class BatchScannerSession method submitTasks.

/**
 */
protected void submitTasks() {
    List<ScannerChunk> newChunks;
    newChunks = Lists.newArrayList(currentBatch);
    currentBatch.clear();
    Collections.shuffle(newChunks);
    for (ScannerChunk chunk : newChunks) {
        AtomicInteger numChunks = serverMap.get(chunk.getLastKnownLocation());
        if (numChunks == null) {
            numChunks = new AtomicInteger(1);
            serverMap.put(chunk.getLastKnownLocation(), numChunks);
        } else
            numChunks.incrementAndGet();
        Scan scan = null;
        if (speculativeScanning && delegatedResourceInitializer == RfileResource.class) {
            if (log.isTraceEnabled()) {
                log.trace("Using speculative execution");
            }
            scan = new SpeculativeScan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
            ((SpeculativeScan) scan).addScan(new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, BatchResource.class, ((SpeculativeScan) scan).getQueue(), listenerService));
            ((SpeculativeScan) scan).addScan(new Scan(localTableName, localAuths, new ScannerChunk(chunk), delegatorReference, delegatedResourceInitializer, ((SpeculativeScan) scan).getQueue(), listenerService));
        } else {
            scan = new Scan(localTableName, localAuths, chunk, delegatorReference, delegatedResourceInitializer, resultQueue, listenerService);
        }
        if (backoffEnabled) {
            scan.setSessionArbiter(this);
        }
        scan.setVisitors(visitorFunctions);
        scan.setTimeout(scanLimitTimeout);
        if (log.isTraceEnabled()) {
            log.trace("Adding scanner " + chunk);
        }
        submitScan(scan, true);
    }
}
Also used : RfileResource(datawave.mr.bulk.RfileResource) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SpeculativeScan(datawave.query.tables.async.SpeculativeScan) ScannerChunk(datawave.query.tables.async.ScannerChunk) Scan(datawave.query.tables.async.Scan) SpeculativeScan(datawave.query.tables.async.SpeculativeScan)

Aggregations

RfileResource (datawave.mr.bulk.RfileResource)2 Scan (datawave.query.tables.async.Scan)2 ScannerChunk (datawave.query.tables.async.ScannerChunk)2 SpeculativeScan (datawave.query.tables.async.SpeculativeScan)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2