Examples with YieldCallback org.apache.accumulo.core.iterators.YieldCallback used on opensource projects

Example 1 with YieldCallback

use of org.apache.accumulo.core.iterators.YieldCallback in project accumulo by apache.

the class YieldingTestCase method test.

@Override
public IteratorTestOutput test(IteratorTestInput testInput) {
    final SortedKeyValueIterator<Key, Value> skvi = IteratorTestUtil.instantiateIterator(testInput);
    final SortedKeyValueIterator<Key, Value> source = IteratorTestUtil.createSource(testInput);
    try {
        skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
        YieldCallback<Key> yield = new YieldCallback<>();
        skvi.enableYielding(yield);
        skvi.seek(testInput.getRange(), testInput.getFamilies(), testInput.isInclusive());
        return new IteratorTestOutput(consume(testInput, skvi, yield));
    } catch (IOException e) {
        return new IteratorTestOutput(e);
    }
}

Example 2 with YieldCallback

use of org.apache.accumulo.core.iterators.YieldCallback in project accumulo by apache.

the class Tablet method nextBatch.

Batch nextBatch(SortedKeyValueIterator<Key, Value> iter, Range range, int num, Set<Column> columns, long batchTimeOut, boolean isolated) throws IOException {
    // log.info("In nextBatch..");
    long stopTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
    if (batchTimeOut == Long.MAX_VALUE || batchTimeOut <= 0) {
        batchTimeOut = 0;
    }
    List<KVEntry> results = new ArrayList<>();
    Key key = null;
    Value value;
    long resultSize = 0L;
    long resultBytes = 0L;
    long maxResultsSize = tableConfiguration.getAsBytes(Property.TABLE_SCAN_MAXMEM);
    Key continueKey = null;
    boolean skipContinueKey = false;
    YieldCallback<Key> yield = new YieldCallback<>();
    // we cannot yield if we are in isolation mode
    if (!isolated) {
        iter.enableYielding(yield);
    }
    if (columns.size() == 0) {
        iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
    } else {
        iter.seek(range, LocalityGroupUtil.families(columns), true);
    }
    while (iter.hasTop()) {
        if (yield.hasYielded()) {
            throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
        }
        value = iter.getTopValue();
        key = iter.getTopKey();
        // copies key and value
        KVEntry kvEntry = new KVEntry(key, value);
        results.add(kvEntry);
        resultSize += kvEntry.estimateMemoryUsed();
        resultBytes += kvEntry.numBytes();
        boolean timesUp = batchTimeOut > 0 && System.nanoTime() >= stopTime;
        if (resultSize >= maxResultsSize || results.size() >= num || timesUp) {
            continueKey = new Key(key);
            skipContinueKey = true;
            break;
        }
        iter.next();
    }
    if (yield.hasYielded()) {
        continueKey = new Key(yield.getPositionAndReset());
        skipContinueKey = true;
        if (!range.contains(continueKey)) {
            throw new IOException("Underlying iterator yielded to a position outside of its range: " + continueKey + " not in " + range);
        }
        if (!results.isEmpty() && continueKey.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
            throw new IOException("Underlying iterator yielded to a position that does not follow the last key returned: " + continueKey + " <= " + results.get(results.size() - 1).getKey());
        }
        log.debug("Scan yield detected at position " + continueKey);
        Metrics scanMetrics = getTabletServer().getScanMetrics();
        if (scanMetrics.isEnabled())
            scanMetrics.add(TabletServerScanMetrics.YIELD, 1);
    } else if (!iter.hasTop()) {
        // end of tablet has been reached
        continueKey = null;
        if (results.size() == 0)
            results = null;
    }
    return new Batch(skipContinueKey, results, continueKey, resultBytes);
}

Example 3 with YieldCallback

use of org.apache.accumulo.core.iterators.YieldCallback in project accumulo by apache.

the class Tablet method lookup.

private LookupResult lookup(SortedKeyValueIterator<Key, Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, List<KVEntry> results, long maxResultsSize, long batchTimeOut) throws IOException {
    LookupResult lookupResult = new LookupResult();
    boolean exceededMemoryUsage = false;
    boolean tabletClosed = false;
    Set<ByteSequence> cfset = null;
    if (columnSet.size() > 0)
        cfset = LocalityGroupUtil.families(columnSet);
    long returnTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
    if (batchTimeOut <= 0 || batchTimeOut == Long.MAX_VALUE) {
        batchTimeOut = 0;
    }
    // determine if the iterator supported yielding
    YieldCallback<Key> yield = new YieldCallback<>();
    mmfi.enableYielding(yield);
    boolean yielded = false;
    for (Range range : ranges) {
        boolean timesUp = batchTimeOut > 0 && System.nanoTime() > returnTime;
        if (exceededMemoryUsage || tabletClosed || timesUp || yielded) {
            lookupResult.unfinishedRanges.add(range);
            continue;
        }
        int entriesAdded = 0;
        try {
            if (cfset != null)
                mmfi.seek(range, cfset, true);
            else
                mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
            while (mmfi.hasTop()) {
                if (yield.hasYielded()) {
                    throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
                }
                Key key = mmfi.getTopKey();
                KVEntry kve = new KVEntry(key, mmfi.getTopValue());
                results.add(kve);
                entriesAdded++;
                lookupResult.bytesAdded += kve.estimateMemoryUsed();
                lookupResult.dataSize += kve.numBytes();
                exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
                timesUp = batchTimeOut > 0 && System.nanoTime() > returnTime;
                if (exceededMemoryUsage || timesUp) {
                    addUnfinishedRange(lookupResult, range, key, false);
                    break;
                }
                mmfi.next();
            }
            if (yield.hasYielded()) {
                yielded = true;
                Key yieldPosition = yield.getPositionAndReset();
                if (!range.contains(yieldPosition)) {
                    throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + range);
                }
                if (!results.isEmpty() && yieldPosition.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
                    throw new IOException("Underlying iterator yielded to a position that does not follow the last key returned: " + yieldPosition + " <= " + results.get(results.size() - 1).getKey());
                }
                addUnfinishedRange(lookupResult, range, yieldPosition, false);
                log.debug("Scan yield detected at position " + yieldPosition);
                Metrics scanMetrics = getTabletServer().getScanMetrics();
                if (scanMetrics.isEnabled())
                    scanMetrics.add(TabletServerScanMetrics.YIELD, 1);
            }
        } catch (TooManyFilesException tmfe) {
            // treat this as a closed tablet, and let the client retry
            log.warn("Tablet {} has too many files, batch lookup can not run", getExtent());
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        } catch (IOException ioe) {
            if (shutdownInProgress()) {
                // assume HDFS shutdown hook caused this exception
                log.debug("IOException while shutdown in progress", ioe);
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw ioe;
            }
        } catch (IterationInterruptedException iie) {
            if (isClosed()) {
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw iie;
            }
        } catch (TabletClosedException tce) {
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        }
    }
    return lookupResult;
}

Example 4 with YieldCallback

use of org.apache.accumulo.core.iterators.YieldCallback in project accumulo by apache.

the class SourceSwitchingIteratorTest method testYield.

public void testYield() throws Exception {
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    put(tm1, "r1", "cf1", "cq1", 5, "v1");
    put(tm1, "r1", "cf1", "cq3", 5, "v2");
    put(tm1, "r2", "cf1", "cq1", 5, "v3");
    SortedMapIterator smi = new SortedMapIterator(tm1);
    YieldingIterator ymi = new YieldingIterator(smi);
    TestDataSource tds = new TestDataSource(ymi);
    SourceSwitchingIterator ssi = new SourceSwitchingIterator(tds);
    YieldCallback<Key> yield = new YieldCallback<>();
    ssi.enableYielding(yield);
    Range r = new Range();
    ssi.seek(r, new ArrayList<>(), false);
    r = yield(r, ssi, yield);
    testAndCallNext(ssi, "r1", "cf1", "cq1", 5, "v1", true);
    r = yield(r, ssi, yield);
    testAndCallNext(ssi, "r1", "cf1", "cq3", 5, "v2", true);
    r = yield(r, ssi, yield);
    testAndCallNext(ssi, "r2", "cf1", "cq1", 5, "v3", true);
    r = yield(r, ssi, yield);
    assertFalse(ssi.hasTop());
}