Examples with TooManyFilesException org.apache.accumulo.tserver.TooManyFilesException used on opensource projects

Example 1 with TooManyFilesException

use of org.apache.accumulo.tserver.TooManyFilesException in project accumulo by apache.

the class NextBatchTask method run.

@Override
public void run() {
    final ScanSession scanSession = (ScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || scanSession == null)
            return;
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("User: " + scanSession.getUser() + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
        Tablet tablet = server.getOnlineTablet(scanSession.extent);
        if (tablet == null) {
            addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
            return;
        }
        long t1 = System.currentTimeMillis();
        ScanBatch batch = scanSession.scanner.read();
        long t2 = System.currentTimeMillis();
        scanSession.nbTimes.addStat(t2 - t1);
        // there should only be one thing on the queue at a time, so
        // it should be ok to call add()
        // instead of put()... if add() fails because queue is at
        // capacity it means there is code
        // problem somewhere
        addResult(batch);
    } catch (TabletClosedException e) {
        addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (TooManyFilesException | SampleNotPresentException e) {
        addResult(e);
    } catch (OutOfMemoryError ome) {
        Halt.halt("Ran out of memory scanning " + scanSession.extent + " for " + scanSession.client, 1);
        addResult(ome);
    } catch (Throwable e) {
        log.warn("exception while scanning tablet " + (scanSession == null ? "(unknown)" : scanSession.extent), e);
        addResult(e);
    } finally {
        runState.set(ScanRunState.FINISHED);
        Thread.currentThread().setName(oldThreadName);
    }
}

Example 2 with TooManyFilesException

use of org.apache.accumulo.tserver.TooManyFilesException in project accumulo by apache.

the class Tablet method lookup.

private LookupResult lookup(SortedKeyValueIterator<Key, Value> mmfi, List<Range> ranges, HashSet<Column> columnSet, List<KVEntry> results, long maxResultsSize, long batchTimeOut) throws IOException {
    LookupResult lookupResult = new LookupResult();
    boolean exceededMemoryUsage = false;
    boolean tabletClosed = false;
    Set<ByteSequence> cfset = null;
    if (columnSet.size() > 0)
        cfset = LocalityGroupUtil.families(columnSet);
    long returnTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
    if (batchTimeOut <= 0 || batchTimeOut == Long.MAX_VALUE) {
        batchTimeOut = 0;
    }
    // determine if the iterator supported yielding
    YieldCallback<Key> yield = new YieldCallback<>();
    mmfi.enableYielding(yield);
    boolean yielded = false;
    for (Range range : ranges) {
        boolean timesUp = batchTimeOut > 0 && System.nanoTime() > returnTime;
        if (exceededMemoryUsage || tabletClosed || timesUp || yielded) {
            lookupResult.unfinishedRanges.add(range);
            continue;
        }
        int entriesAdded = 0;
        try {
            if (cfset != null)
                mmfi.seek(range, cfset, true);
            else
                mmfi.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false);
            while (mmfi.hasTop()) {
                if (yield.hasYielded()) {
                    throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
                }
                Key key = mmfi.getTopKey();
                KVEntry kve = new KVEntry(key, mmfi.getTopValue());
                results.add(kve);
                entriesAdded++;
                lookupResult.bytesAdded += kve.estimateMemoryUsed();
                lookupResult.dataSize += kve.numBytes();
                exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
                timesUp = batchTimeOut > 0 && System.nanoTime() > returnTime;
                if (exceededMemoryUsage || timesUp) {
                    addUnfinishedRange(lookupResult, range, key, false);
                    break;
                }
                mmfi.next();
            }
            if (yield.hasYielded()) {
                yielded = true;
                Key yieldPosition = yield.getPositionAndReset();
                if (!range.contains(yieldPosition)) {
                    throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + range);
                }
                if (!results.isEmpty() && yieldPosition.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
                    throw new IOException("Underlying iterator yielded to a position that does not follow the last key returned: " + yieldPosition + " <= " + results.get(results.size() - 1).getKey());
                }
                addUnfinishedRange(lookupResult, range, yieldPosition, false);
                log.debug("Scan yield detected at position " + yieldPosition);
                Metrics scanMetrics = getTabletServer().getScanMetrics();
                if (scanMetrics.isEnabled())
                    scanMetrics.add(TabletServerScanMetrics.YIELD, 1);
            }
        } catch (TooManyFilesException tmfe) {
            // treat this as a closed tablet, and let the client retry
            log.warn("Tablet {} has too many files, batch lookup can not run", getExtent());
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        } catch (IOException ioe) {
            if (shutdownInProgress()) {
                // assume HDFS shutdown hook caused this exception
                log.debug("IOException while shutdown in progress", ioe);
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw ioe;
            }
        } catch (IterationInterruptedException iie) {
            if (isClosed()) {
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw iie;
            }
        } catch (TabletClosedException tce) {
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        }
    }
    return lookupResult;
}