Examples with TooManyFilesException org.apache.accumulo.server.fs.TooManyFilesException used on opensource projects

Example 1 with TooManyFilesException

use of org.apache.accumulo.server.fs.TooManyFilesException in project accumulo by apache.

the class ThriftClientHandler method continueScan.

private ScanResult continueScan(TInfo tinfo, long scanID, SingleScanSession scanSession) throws NoSuchScanIDException, NotServingTabletException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException, TSampleNotPresentException {
    if (scanSession.nextBatchTask == null) {
        scanSession.nextBatchTask = new NextBatchTask(server, scanID, scanSession.interruptFlag);
        server.resourceManager.executeReadAhead(scanSession.extent, getScanDispatcher(scanSession.extent), scanSession, scanSession.nextBatchTask);
    }
    ScanBatch bresult;
    try {
        bresult = scanSession.nextBatchTask.get(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
        scanSession.nextBatchTask = null;
    } catch (ExecutionException e) {
        server.sessionManager.removeSession(scanID);
        if (e.getCause() instanceof NotServingTabletException) {
            throw (NotServingTabletException) e.getCause();
        } else if (e.getCause() instanceof TooManyFilesException) {
            throw new org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException(scanSession.extent.toThrift());
        } else if (e.getCause() instanceof SampleNotPresentException) {
            throw new TSampleNotPresentException(scanSession.extent.toThrift());
        } else if (e.getCause() instanceof IOException) {
            sleepUninterruptibly(MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS, TimeUnit.MILLISECONDS);
            List<KVEntry> empty = Collections.emptyList();
            bresult = new ScanBatch(empty, true);
            scanSession.nextBatchTask = null;
        } else {
            throw new RuntimeException(e);
        }
    } catch (CancellationException ce) {
        server.sessionManager.removeSession(scanID);
        Tablet tablet = server.getOnlineTablet(scanSession.extent);
        if (tablet == null || tablet.isClosed()) {
            throw new NotServingTabletException(scanSession.extent.toThrift());
        } else {
            throw new NoSuchScanIDException();
        }
    } catch (TimeoutException e) {
        List<TKeyValue> param = Collections.emptyList();
        long timeout = server.getConfiguration().getTimeInMillis(Property.TSERV_CLIENT_TIMEOUT);
        server.sessionManager.removeIfNotAccessed(scanID, timeout);
        return new ScanResult(param, true);
    } catch (Exception t) {
        server.sessionManager.removeSession(scanID);
        log.warn("Failed to get next batch", t);
        throw new RuntimeException(t);
    }
    ScanResult scanResult = new ScanResult(Key.compress(bresult.getResults()), bresult.isMore());
    scanSession.entriesReturned += scanResult.results.size();
    scanSession.batchCount++;
    if (scanResult.more && scanSession.batchCount > scanSession.readaheadThreshold) {
        // start reading next batch while current batch is transmitted
        // to client
        scanSession.nextBatchTask = new NextBatchTask(server, scanID, scanSession.interruptFlag);
        server.resourceManager.executeReadAhead(scanSession.extent, getScanDispatcher(scanSession.extent), scanSession, scanSession.nextBatchTask);
    }
    if (!scanResult.more) {
        closeScan(tinfo, scanID);
    }
    return scanResult;
}

Example 2 with TooManyFilesException

use of org.apache.accumulo.server.fs.TooManyFilesException in project accumulo by apache.

the class Tablet method lookup.

private LookupResult lookup(SortedKeyValueIterator<Key, Value> mmfi, List<Range> ranges, List<KVEntry> results, ScanParameters scanParams, long maxResultsSize) throws IOException {
    LookupResult lookupResult = new LookupResult();
    boolean exceededMemoryUsage = false;
    boolean tabletClosed = false;
    Set<ByteSequence> cfset = null;
    if (!scanParams.getColumnSet().isEmpty()) {
        cfset = LocalityGroupUtil.families(scanParams.getColumnSet());
    }
    long batchTimeOut = scanParams.getBatchTimeOut();
    long timeToRun = TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
    long startNanos = System.nanoTime();
    if (batchTimeOut <= 0 || batchTimeOut == Long.MAX_VALUE) {
        batchTimeOut = 0;
    }
    // determine if the iterator supported yielding
    YieldCallback<Key> yield = new YieldCallback<>();
    mmfi.enableYielding(yield);
    boolean yielded = false;
    for (Range range : ranges) {
        boolean timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
        if (exceededMemoryUsage || tabletClosed || timesUp || yielded) {
            lookupResult.unfinishedRanges.add(range);
            continue;
        }
        int entriesAdded = 0;
        try {
            if (cfset != null) {
                mmfi.seek(range, cfset, true);
            } else {
                mmfi.seek(range, Set.of(), false);
            }
            while (mmfi.hasTop()) {
                if (yield.hasYielded()) {
                    throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
                }
                Key key = mmfi.getTopKey();
                KVEntry kve = new KVEntry(key, mmfi.getTopValue());
                results.add(kve);
                entriesAdded++;
                lookupResult.bytesAdded += kve.estimateMemoryUsed();
                lookupResult.dataSize += kve.numBytes();
                exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
                timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
                if (exceededMemoryUsage || timesUp) {
                    addUnfinishedRange(lookupResult, range, key);
                    break;
                }
                mmfi.next();
            }
            if (yield.hasYielded()) {
                yielded = true;
                Key yieldPosition = yield.getPositionAndReset();
                if (!range.contains(yieldPosition)) {
                    throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + range);
                }
                if (!results.isEmpty() && yieldPosition.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
                    throw new IOException("Underlying iterator yielded to a position" + " that does not follow the last key returned: " + yieldPosition + " <= " + results.get(results.size() - 1).getKey());
                }
                addUnfinishedRange(lookupResult, range, yieldPosition);
                log.debug("Scan yield detected at position " + yieldPosition);
                getTabletServer().getScanMetrics().addYield(1);
            }
        } catch (TooManyFilesException tmfe) {
            // treat this as a closed tablet, and let the client retry
            log.warn("Tablet {} has too many files, batch lookup can not run", getExtent());
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        } catch (IOException ioe) {
            if (ShutdownUtil.isShutdownInProgress()) {
                // assume HDFS shutdown hook caused this exception
                log.debug("IOException while shutdown in progress", ioe);
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw ioe;
            }
        } catch (IterationInterruptedException iie) {
            if (isClosed()) {
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw iie;
            }
        } catch (TabletClosedException tce) {
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        }
    }
    return lookupResult;
}

Example 3 with TooManyFilesException

use of org.apache.accumulo.server.fs.TooManyFilesException in project accumulo by apache.

the class ThriftClientHandler method checkConditions.

private void checkConditions(Map<KeyExtent, List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs, List<String> symbols) throws IOException {
    Iterator<Entry<KeyExtent, List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
    final CompressedIterators compressedIters = new CompressedIterators(symbols);
    ConditionCheckerContext checkerContext = new ConditionCheckerContext(server.getContext(), compressedIters, context.getTableConfiguration(cs.tableId));
    while (iter.hasNext()) {
        final Entry<KeyExtent, List<ServerConditionalMutation>> entry = iter.next();
        final Tablet tablet = server.getOnlineTablet(entry.getKey());
        if (tablet == null || tablet.isClosed()) {
            for (ServerConditionalMutation scm : entry.getValue()) {
                results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
            }
            iter.remove();
        } else {
            final List<ServerConditionalMutation> okMutations = new ArrayList<>(entry.getValue().size());
            final List<TCMResult> resultsSubList = results.subList(results.size(), results.size());
            ConditionChecker checker = checkerContext.newChecker(entry.getValue(), okMutations, resultsSubList);
            try {
                tablet.checkConditions(checker, cs.auths, cs.interruptFlag);
                if (okMutations.isEmpty()) {
                    iter.remove();
                } else {
                    entry.setValue(okMutations);
                }
            } catch (TabletClosedException | IterationInterruptedException | TooManyFilesException e) {
                // clear anything added while checking conditions.
                resultsSubList.clear();
                for (ServerConditionalMutation scm : entry.getValue()) {
                    results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
                }
                iter.remove();
            }
        }
    }
}

Example 4 with TooManyFilesException

use of org.apache.accumulo.server.fs.TooManyFilesException in project accumulo by apache.

the class NextBatchTask method run.

@Override
public void run() {
    final SingleScanSession scanSession = (SingleScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || scanSession == null)
            return;
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("User: " + scanSession.getUser() + " Start: " + scanSession.startTime + " Client: " + scanSession.client + " Tablet: " + scanSession.extent);
        Tablet tablet = server.getOnlineTablet(scanSession.extent);
        if (tablet == null) {
            addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
            return;
        }
        ScanBatch batch = scanSession.scanner.read();
        // there should only be one thing on the queue at a time, so
        // it should be ok to call add()
        // instead of put()... if add() fails because queue is at
        // capacity it means there is code
        // problem somewhere
        addResult(batch);
    } catch (TabletClosedException e) {
        addResult(new org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException(scanSession.extent.toThrift()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (TooManyFilesException | SampleNotPresentException e) {
        addResult(e);
    } catch (IOException | RuntimeException e) {
        log.warn("exception while scanning tablet {} for {}", scanSession.extent, scanSession.client, e);
        addResult(e);
    } finally {
        runState.set(ScanRunState.FINISHED);
        Thread.currentThread().setName(oldThreadName);
    }
}