Examples with IterationInterruptedException org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException used on opensource projects

Example 1 with IterationInterruptedException

use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.

the class Tablet method lookup.

private LookupResult lookup(SortedKeyValueIterator<Key, Value> mmfi, List<Range> ranges, List<KVEntry> results, ScanParameters scanParams, long maxResultsSize) throws IOException {
    LookupResult lookupResult = new LookupResult();
    boolean exceededMemoryUsage = false;
    boolean tabletClosed = false;
    Set<ByteSequence> cfset = null;
    if (!scanParams.getColumnSet().isEmpty()) {
        cfset = LocalityGroupUtil.families(scanParams.getColumnSet());
    }
    long batchTimeOut = scanParams.getBatchTimeOut();
    long timeToRun = TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
    long startNanos = System.nanoTime();
    if (batchTimeOut <= 0 || batchTimeOut == Long.MAX_VALUE) {
        batchTimeOut = 0;
    }
    // determine if the iterator supported yielding
    YieldCallback<Key> yield = new YieldCallback<>();
    mmfi.enableYielding(yield);
    boolean yielded = false;
    for (Range range : ranges) {
        boolean timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
        if (exceededMemoryUsage || tabletClosed || timesUp || yielded) {
            lookupResult.unfinishedRanges.add(range);
            continue;
        }
        int entriesAdded = 0;
        try {
            if (cfset != null) {
                mmfi.seek(range, cfset, true);
            } else {
                mmfi.seek(range, Set.of(), false);
            }
            while (mmfi.hasTop()) {
                if (yield.hasYielded()) {
                    throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
                }
                Key key = mmfi.getTopKey();
                KVEntry kve = new KVEntry(key, mmfi.getTopValue());
                results.add(kve);
                entriesAdded++;
                lookupResult.bytesAdded += kve.estimateMemoryUsed();
                lookupResult.dataSize += kve.numBytes();
                exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
                timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
                if (exceededMemoryUsage || timesUp) {
                    addUnfinishedRange(lookupResult, range, key);
                    break;
                }
                mmfi.next();
            }
            if (yield.hasYielded()) {
                yielded = true;
                Key yieldPosition = yield.getPositionAndReset();
                if (!range.contains(yieldPosition)) {
                    throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + range);
                }
                if (!results.isEmpty() && yieldPosition.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
                    throw new IOException("Underlying iterator yielded to a position" + " that does not follow the last key returned: " + yieldPosition + " <= " + results.get(results.size() - 1).getKey());
                }
                addUnfinishedRange(lookupResult, range, yieldPosition);
                log.debug("Scan yield detected at position " + yieldPosition);
                getTabletServer().getScanMetrics().addYield(1);
            }
        } catch (TooManyFilesException tmfe) {
            // treat this as a closed tablet, and let the client retry
            log.warn("Tablet {} has too many files, batch lookup can not run", getExtent());
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        } catch (IOException ioe) {
            if (ShutdownUtil.isShutdownInProgress()) {
                // assume HDFS shutdown hook caused this exception
                log.debug("IOException while shutdown in progress", ioe);
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw ioe;
            }
        } catch (IterationInterruptedException iie) {
            if (isClosed()) {
                handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
                tabletClosed = true;
            } else {
                throw iie;
            }
        } catch (TabletClosedException tce) {
            handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
            tabletClosed = true;
        }
    }
    return lookupResult;
}

Example 2 with IterationInterruptedException

use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.

the class ScanDataSource method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
    Map<TabletFile, DataFileValue> files;
    SamplerConfigurationImpl samplerConfig = scanParams.getSamplerConfigurationImpl();
    synchronized (tablet) {
        if (memIters != null)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
        if (tablet.isClosed())
            throw new TabletClosedException();
        if (interruptFlag.get())
            throw new IterationInterruptedException(tablet.getExtent() + " " + interruptFlag.hashCode());
        // only acquire the file manager when we know the tablet is open
        if (fileManager == null) {
            fileManager = tablet.getTabletResources().newScanFileManager(scanParams.getScanDispatch());
            tablet.getTabletServer().getScanMetrics().incrementOpenFiles(fileManager.getNumOpenFiles());
            tablet.addActiveScans(this);
        }
        if (fileManager.getNumOpenFiles() != 0)
            throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
        // set this before trying to get iterators in case
        // getIterators() throws an exception
        expectedDeletionCount = tablet.getDataSourceDeletions();
        memIters = tablet.getTabletMemory().getIterators(samplerConfig);
        Pair<Long, Map<TabletFile, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
        fileReservationId = reservation.getFirst();
        files = reservation.getSecond();
    }
    Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, scanParams.isIsolated(), samplerConfig);
    for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
    iters.addAll(mapfiles);
    iters.addAll(memIters);
    MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
    TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(tablet.getTabletServer().getContext(), IteratorScope.scan, tablet.getTableConfiguration(), tablet.getExtent().tableId(), fileManager, files, scanParams.getAuthorizations(), samplerConfig, new ArrayList<>());
    statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
    SortedKeyValueIterator<Key, Value> visFilter = SystemIteratorUtil.setupSystemScanIterators(statsIterator, scanParams.getColumnSet(), scanParams.getAuthorizations(), defaultLabels, tablet.getTableConfiguration());
    if (loadIters) {
        List<IterInfo> iterInfos;
        Map<String, Map<String, String>> iterOpts;
        ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
        if (scanParams.getSsiList().isEmpty() && scanParams.getSsio().isEmpty()) {
            // No scan time iterator options were set, so can just use the pre-parsed table iterator
            // options.
            iterInfos = pic.getIterInfo();
            iterOpts = pic.getOpts();
        } else {
            // Scan time iterator options were set, so need to merge those with pre-parsed table
            // iterator options.
            iterOpts = new HashMap<>(pic.getOpts().size() + scanParams.getSsio().size());
            iterInfos = new ArrayList<>(pic.getIterInfo().size() + scanParams.getSsiList().size());
            IterConfigUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), scanParams.getSsiList(), scanParams.getSsio());
        }
        String context;
        if (scanParams.getClassLoaderContext() != null) {
            log.trace("Loading iterators for scan with scan context: {}", scanParams.getClassLoaderContext());
            context = scanParams.getClassLoaderContext();
        } else {
            context = pic.getServiceEnv();
            if (context != null) {
                log.trace("Loading iterators for scan with table context: {}", scanParams.getClassLoaderContext());
            } else {
                log.trace("Loading iterators for scan");
            }
        }
        IterLoad il = new IterLoad().iters(iterInfos).iterOpts(iterOpts).iterEnv(iterEnv).useAccumuloClassLoader(true).context(context);
        return iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter, il));
    } else {
        return visFilter;
    }
}

Example 3 with IterationInterruptedException

use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.

the class Scanner method read.

public ScanBatch read() throws IOException, TabletClosedException {
    ScanDataSource dataSource = null;
    Batch results = null;
    try {
        try {
            scannerSemaphore.acquire();
        } catch (InterruptedException e) {
            sawException = true;
        }
        // exception was its cause
        if (sawException)
            throw new IllegalStateException("Tried to use scanner after exception occurred.");
        if (scanClosed)
            throw new IllegalStateException("Tried to use scanner after it was closed.");
        if (scanParams.isIsolated()) {
            if (isolatedDataSource == null)
                isolatedDataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
            dataSource = isolatedDataSource;
        } else {
            dataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
        }
        SortedKeyValueIterator<Key, Value> iter;
        if (scanParams.isIsolated()) {
            if (isolatedIter == null)
                isolatedIter = new SourceSwitchingIterator(dataSource, true);
            else
                isolatedDataSource.reattachFileManager();
            iter = isolatedIter;
        } else {
            iter = new SourceSwitchingIterator(dataSource, false);
        }
        results = tablet.nextBatch(iter, range, scanParams);
        if (results.getResults() == null) {
            range = null;
            return new ScanBatch(new ArrayList<>(), false);
        } else if (results.getContinueKey() == null) {
            return new ScanBatch(results.getResults(), false);
        } else {
            range = new Range(results.getContinueKey(), !results.isSkipContinueKey(), range.getEndKey(), range.isEndKeyInclusive());
            return new ScanBatch(results.getResults(), true);
        }
    } catch (IterationInterruptedException iie) {
        sawException = true;
        if (tablet.isClosed())
            throw new TabletClosedException(iie);
        else
            throw iie;
    } catch (IOException ioe) {
        if (ShutdownUtil.isShutdownInProgress()) {
            log.debug("IOException while shutdown in progress ", ioe);
            // assume IOException was caused by execution of HDFS
            throw new TabletClosedException(ioe);
        // shutdown hook
        }
        sawException = true;
        dataSource.close(true);
        throw ioe;
    } catch (RuntimeException re) {
        sawException = true;
        throw re;
    } finally {
        // to return mapfiles, even when exception is thrown
        if (dataSource != null && !scanParams.isIsolated()) {
            dataSource.close(false);
        } else if (dataSource != null) {
            dataSource.detachFileManager();
        }
        if (results != null && results.getResults() != null)
            tablet.updateQueryStats(results.getResults().size(), results.getNumBytes());
        scannerSemaphore.release();
    }
}

Example 4 with IterationInterruptedException

use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.

the class ThriftClientHandler method checkConditions.

private void checkConditions(Map<KeyExtent, List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs, List<String> symbols) throws IOException {
    Iterator<Entry<KeyExtent, List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
    final CompressedIterators compressedIters = new CompressedIterators(symbols);
    ConditionCheckerContext checkerContext = new ConditionCheckerContext(server.getContext(), compressedIters, context.getTableConfiguration(cs.tableId));
    while (iter.hasNext()) {
        final Entry<KeyExtent, List<ServerConditionalMutation>> entry = iter.next();
        final Tablet tablet = server.getOnlineTablet(entry.getKey());
        if (tablet == null || tablet.isClosed()) {
            for (ServerConditionalMutation scm : entry.getValue()) {
                results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
            }
            iter.remove();
        } else {
            final List<ServerConditionalMutation> okMutations = new ArrayList<>(entry.getValue().size());
            final List<TCMResult> resultsSubList = results.subList(results.size(), results.size());
            ConditionChecker checker = checkerContext.newChecker(entry.getValue(), okMutations, resultsSubList);
            try {
                tablet.checkConditions(checker, cs.auths, cs.interruptFlag);
                if (okMutations.isEmpty()) {
                    iter.remove();
                } else {
                    entry.setValue(okMutations);
                }
            } catch (TabletClosedException | IterationInterruptedException | TooManyFilesException e) {
                // clear anything added while checking conditions.
                resultsSubList.clear();
                for (ServerConditionalMutation scm : entry.getValue()) {
                    results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
                }
                iter.remove();
            }
        }
    }
}

Example 5 with IterationInterruptedException

use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.isEmpty()) {
                fullScans.add(entry.getKey());
            } else {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        // @formatter:off
        Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
        // @formatter:on
        List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Exception e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}