use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.
the class Tablet method lookup.
private LookupResult lookup(SortedKeyValueIterator<Key, Value> mmfi, List<Range> ranges, List<KVEntry> results, ScanParameters scanParams, long maxResultsSize) throws IOException {
LookupResult lookupResult = new LookupResult();
boolean exceededMemoryUsage = false;
boolean tabletClosed = false;
Set<ByteSequence> cfset = null;
if (!scanParams.getColumnSet().isEmpty()) {
cfset = LocalityGroupUtil.families(scanParams.getColumnSet());
}
long batchTimeOut = scanParams.getBatchTimeOut();
long timeToRun = TimeUnit.MILLISECONDS.toNanos(batchTimeOut);
long startNanos = System.nanoTime();
if (batchTimeOut <= 0 || batchTimeOut == Long.MAX_VALUE) {
batchTimeOut = 0;
}
// determine if the iterator supported yielding
YieldCallback<Key> yield = new YieldCallback<>();
mmfi.enableYielding(yield);
boolean yielded = false;
for (Range range : ranges) {
boolean timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
if (exceededMemoryUsage || tabletClosed || timesUp || yielded) {
lookupResult.unfinishedRanges.add(range);
continue;
}
int entriesAdded = 0;
try {
if (cfset != null) {
mmfi.seek(range, cfset, true);
} else {
mmfi.seek(range, Set.of(), false);
}
while (mmfi.hasTop()) {
if (yield.hasYielded()) {
throw new IOException("Coding error: hasTop returned true but has yielded at " + yield.getPositionAndReset());
}
Key key = mmfi.getTopKey();
KVEntry kve = new KVEntry(key, mmfi.getTopValue());
results.add(kve);
entriesAdded++;
lookupResult.bytesAdded += kve.estimateMemoryUsed();
lookupResult.dataSize += kve.numBytes();
exceededMemoryUsage = lookupResult.bytesAdded > maxResultsSize;
timesUp = batchTimeOut > 0 && (System.nanoTime() - startNanos) > timeToRun;
if (exceededMemoryUsage || timesUp) {
addUnfinishedRange(lookupResult, range, key);
break;
}
mmfi.next();
}
if (yield.hasYielded()) {
yielded = true;
Key yieldPosition = yield.getPositionAndReset();
if (!range.contains(yieldPosition)) {
throw new IOException("Underlying iterator yielded to a position outside of its range: " + yieldPosition + " not in " + range);
}
if (!results.isEmpty() && yieldPosition.compareTo(results.get(results.size() - 1).getKey()) <= 0) {
throw new IOException("Underlying iterator yielded to a position" + " that does not follow the last key returned: " + yieldPosition + " <= " + results.get(results.size() - 1).getKey());
}
addUnfinishedRange(lookupResult, range, yieldPosition);
log.debug("Scan yield detected at position " + yieldPosition);
getTabletServer().getScanMetrics().addYield(1);
}
} catch (TooManyFilesException tmfe) {
// treat this as a closed tablet, and let the client retry
log.warn("Tablet {} has too many files, batch lookup can not run", getExtent());
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
tabletClosed = true;
} catch (IOException ioe) {
if (ShutdownUtil.isShutdownInProgress()) {
// assume HDFS shutdown hook caused this exception
log.debug("IOException while shutdown in progress", ioe);
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
tabletClosed = true;
} else {
throw ioe;
}
} catch (IterationInterruptedException iie) {
if (isClosed()) {
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
tabletClosed = true;
} else {
throw iie;
}
} catch (TabletClosedException tce) {
handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range, entriesAdded);
tabletClosed = true;
}
}
return lookupResult;
}
use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.
the class ScanDataSource method createIterator.
private SortedKeyValueIterator<Key, Value> createIterator() throws IOException {
Map<TabletFile, DataFileValue> files;
SamplerConfigurationImpl samplerConfig = scanParams.getSamplerConfigurationImpl();
synchronized (tablet) {
if (memIters != null)
throw new IllegalStateException("Tried to create new scan iterator w/o releasing memory");
if (tablet.isClosed())
throw new TabletClosedException();
if (interruptFlag.get())
throw new IterationInterruptedException(tablet.getExtent() + " " + interruptFlag.hashCode());
// only acquire the file manager when we know the tablet is open
if (fileManager == null) {
fileManager = tablet.getTabletResources().newScanFileManager(scanParams.getScanDispatch());
tablet.getTabletServer().getScanMetrics().incrementOpenFiles(fileManager.getNumOpenFiles());
tablet.addActiveScans(this);
}
if (fileManager.getNumOpenFiles() != 0)
throw new IllegalStateException("Tried to create new scan iterator w/o releasing files");
// set this before trying to get iterators in case
// getIterators() throws an exception
expectedDeletionCount = tablet.getDataSourceDeletions();
memIters = tablet.getTabletMemory().getIterators(samplerConfig);
Pair<Long, Map<TabletFile, DataFileValue>> reservation = tablet.getDatafileManager().reserveFilesForScan();
fileReservationId = reservation.getFirst();
files = reservation.getSecond();
}
Collection<InterruptibleIterator> mapfiles = fileManager.openFiles(files, scanParams.isIsolated(), samplerConfig);
for (SortedKeyValueIterator<Key, Value> skvi : Iterables.concat(mapfiles, memIters)) ((InterruptibleIterator) skvi).setInterruptFlag(interruptFlag);
List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(mapfiles.size() + memIters.size());
iters.addAll(mapfiles);
iters.addAll(memIters);
MultiIterator multiIter = new MultiIterator(iters, tablet.getExtent());
TabletIteratorEnvironment iterEnv = new TabletIteratorEnvironment(tablet.getTabletServer().getContext(), IteratorScope.scan, tablet.getTableConfiguration(), tablet.getExtent().tableId(), fileManager, files, scanParams.getAuthorizations(), samplerConfig, new ArrayList<>());
statsIterator = new StatsIterator(multiIter, TabletServer.seekCount, tablet.getScannedCounter());
SortedKeyValueIterator<Key, Value> visFilter = SystemIteratorUtil.setupSystemScanIterators(statsIterator, scanParams.getColumnSet(), scanParams.getAuthorizations(), defaultLabels, tablet.getTableConfiguration());
if (loadIters) {
List<IterInfo> iterInfos;
Map<String, Map<String, String>> iterOpts;
ParsedIteratorConfig pic = tablet.getTableConfiguration().getParsedIteratorConfig(IteratorScope.scan);
if (scanParams.getSsiList().isEmpty() && scanParams.getSsio().isEmpty()) {
// No scan time iterator options were set, so can just use the pre-parsed table iterator
// options.
iterInfos = pic.getIterInfo();
iterOpts = pic.getOpts();
} else {
// Scan time iterator options were set, so need to merge those with pre-parsed table
// iterator options.
iterOpts = new HashMap<>(pic.getOpts().size() + scanParams.getSsio().size());
iterInfos = new ArrayList<>(pic.getIterInfo().size() + scanParams.getSsiList().size());
IterConfigUtil.mergeIteratorConfig(iterInfos, iterOpts, pic.getIterInfo(), pic.getOpts(), scanParams.getSsiList(), scanParams.getSsio());
}
String context;
if (scanParams.getClassLoaderContext() != null) {
log.trace("Loading iterators for scan with scan context: {}", scanParams.getClassLoaderContext());
context = scanParams.getClassLoaderContext();
} else {
context = pic.getServiceEnv();
if (context != null) {
log.trace("Loading iterators for scan with table context: {}", scanParams.getClassLoaderContext());
} else {
log.trace("Loading iterators for scan");
}
}
IterLoad il = new IterLoad().iters(iterInfos).iterOpts(iterOpts).iterEnv(iterEnv).useAccumuloClassLoader(true).context(context);
return iterEnv.getTopLevelIterator(IterConfigUtil.loadIterators(visFilter, il));
} else {
return visFilter;
}
}
use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.
the class Scanner method read.
public ScanBatch read() throws IOException, TabletClosedException {
ScanDataSource dataSource = null;
Batch results = null;
try {
try {
scannerSemaphore.acquire();
} catch (InterruptedException e) {
sawException = true;
}
// exception was its cause
if (sawException)
throw new IllegalStateException("Tried to use scanner after exception occurred.");
if (scanClosed)
throw new IllegalStateException("Tried to use scanner after it was closed.");
if (scanParams.isIsolated()) {
if (isolatedDataSource == null)
isolatedDataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
dataSource = isolatedDataSource;
} else {
dataSource = new ScanDataSource(tablet, scanParams, true, interruptFlag);
}
SortedKeyValueIterator<Key, Value> iter;
if (scanParams.isIsolated()) {
if (isolatedIter == null)
isolatedIter = new SourceSwitchingIterator(dataSource, true);
else
isolatedDataSource.reattachFileManager();
iter = isolatedIter;
} else {
iter = new SourceSwitchingIterator(dataSource, false);
}
results = tablet.nextBatch(iter, range, scanParams);
if (results.getResults() == null) {
range = null;
return new ScanBatch(new ArrayList<>(), false);
} else if (results.getContinueKey() == null) {
return new ScanBatch(results.getResults(), false);
} else {
range = new Range(results.getContinueKey(), !results.isSkipContinueKey(), range.getEndKey(), range.isEndKeyInclusive());
return new ScanBatch(results.getResults(), true);
}
} catch (IterationInterruptedException iie) {
sawException = true;
if (tablet.isClosed())
throw new TabletClosedException(iie);
else
throw iie;
} catch (IOException ioe) {
if (ShutdownUtil.isShutdownInProgress()) {
log.debug("IOException while shutdown in progress ", ioe);
// assume IOException was caused by execution of HDFS
throw new TabletClosedException(ioe);
// shutdown hook
}
sawException = true;
dataSource.close(true);
throw ioe;
} catch (RuntimeException re) {
sawException = true;
throw re;
} finally {
// to return mapfiles, even when exception is thrown
if (dataSource != null && !scanParams.isIsolated()) {
dataSource.close(false);
} else if (dataSource != null) {
dataSource.detachFileManager();
}
if (results != null && results.getResults() != null)
tablet.updateQueryStats(results.getResults().size(), results.getNumBytes());
scannerSemaphore.release();
}
}
use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.
the class ThriftClientHandler method checkConditions.
private void checkConditions(Map<KeyExtent, List<ServerConditionalMutation>> updates, ArrayList<TCMResult> results, ConditionalSession cs, List<String> symbols) throws IOException {
Iterator<Entry<KeyExtent, List<ServerConditionalMutation>>> iter = updates.entrySet().iterator();
final CompressedIterators compressedIters = new CompressedIterators(symbols);
ConditionCheckerContext checkerContext = new ConditionCheckerContext(server.getContext(), compressedIters, context.getTableConfiguration(cs.tableId));
while (iter.hasNext()) {
final Entry<KeyExtent, List<ServerConditionalMutation>> entry = iter.next();
final Tablet tablet = server.getOnlineTablet(entry.getKey());
if (tablet == null || tablet.isClosed()) {
for (ServerConditionalMutation scm : entry.getValue()) {
results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
}
iter.remove();
} else {
final List<ServerConditionalMutation> okMutations = new ArrayList<>(entry.getValue().size());
final List<TCMResult> resultsSubList = results.subList(results.size(), results.size());
ConditionChecker checker = checkerContext.newChecker(entry.getValue(), okMutations, resultsSubList);
try {
tablet.checkConditions(checker, cs.auths, cs.interruptFlag);
if (okMutations.isEmpty()) {
iter.remove();
} else {
entry.setValue(okMutations);
}
} catch (TabletClosedException | IterationInterruptedException | TooManyFilesException e) {
// clear anything added while checking conditions.
resultsSubList.clear();
for (ServerConditionalMutation scm : entry.getValue()) {
results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED));
}
iter.remove();
}
}
}
}
use of org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException in project accumulo by apache.
the class LookupTask method run.
@Override
public void run() {
MultiScanSession session = (MultiScanSession) server.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
try {
if (isCancelled() || session == null)
return;
TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
runState.set(ScanRunState.RUNNING);
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
long bytesAdded = 0;
long maxScanTime = 4000;
long startTime = System.currentTimeMillis();
List<KVEntry> results = new ArrayList<>();
Map<KeyExtent, List<Range>> failures = new HashMap<>();
List<KeyExtent> fullScans = new ArrayList<>();
KeyExtent partScan = null;
Key partNextKey = null;
boolean partNextKeyInclusive = false;
Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
// check the time so that the read ahead thread is not monopolized
while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
Entry<KeyExtent, List<Range>> entry = iter.next();
iter.remove();
// check that tablet server is serving requested tablet
Tablet tablet = server.getOnlineTablet(entry.getKey());
if (tablet == null) {
failures.put(entry.getKey(), entry.getValue());
continue;
}
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
LookupResult lookupResult;
try {
// canceled
if (isCancelled())
interruptFlag.set(true);
lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
// if the tablet was closed it it possible that the
// interrupt flag was set.... do not want it set for
// the next
// lookup
interruptFlag.set(false);
} catch (IOException e) {
log.warn("lookup failed for tablet " + entry.getKey(), e);
throw new RuntimeException(e);
}
bytesAdded += lookupResult.bytesAdded;
if (lookupResult.unfinishedRanges.isEmpty()) {
fullScans.add(entry.getKey());
} else {
if (lookupResult.closed) {
failures.put(entry.getKey(), lookupResult.unfinishedRanges);
} else {
session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
partScan = entry.getKey();
partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
}
}
}
long finishTime = System.currentTimeMillis();
session.totalLookupTime += (finishTime - startTime);
session.numEntries += results.size();
// convert everything to thrift before adding result
List<TKeyValue> retResults = new ArrayList<>();
for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
// @formatter:off
Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
// @formatter:on
List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
TKeyExtent retPartScan = null;
TKey retPartNextKey = null;
if (partScan != null) {
retPartScan = partScan.toThrift();
retPartNextKey = partNextKey.toThrift();
}
// add results to queue
addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
} catch (IterationInterruptedException iie) {
if (!isCancelled()) {
log.warn("Iteration interrupted, when scan not cancelled", iie);
addResult(iie);
}
} catch (SampleNotPresentException e) {
addResult(e);
} catch (Exception e) {
log.warn("exception while doing multi-scan ", e);
addResult(e);
} finally {
Thread.currentThread().setName(oldThreadName);
runState.set(ScanRunState.FINISHED);
}
}
Aggregations