use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.
the class TabletServerBatchReaderIterator method doLookup.
static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
if (requested.isEmpty()) {
return;
}
// copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
ArrayList<Range> ranges = new ArrayList<>();
for (Range range : entry.getValue()) {
ranges.add(new Range(range));
}
unscanned.put(KeyExtent.copyOf(entry.getKey()), ranges);
}
timeoutTracker.startingScan();
try {
final HostAndPort parsedServer = HostAndPort.fromString(server);
final TabletClientService.Client client;
if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
else
client = ThriftUtil.getTServerClient(parsedServer, context);
try {
OpTimer timer = null;
if (log.isTraceEnabled()) {
log.trace("tid={} Starting multi scan, tserver={} #tablets={} #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
timer = new OpTimer().start();
}
TabletType ttype = TabletType.type(requested.keySet());
boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
// @formatter:off
Map<TKeyExtent, List<TRange>> thriftTabletRanges = requested.entrySet().stream().collect(Collectors.toMap((entry) -> entry.getKey().toThrift(), (entry) -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
// @formatter:on
Map<String, String> execHints = options.executionHints.isEmpty() ? null : options.executionHints;
InitialMultiScan imsr = client.startMultiScan(TraceUtil.traceInfo(), context.rpcCreds(), thriftTabletRanges, columns.stream().map(Column::toThrift).collect(Collectors.toList()), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext, execHints);
if (waitForWrites)
ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
MultiScanResult scanResult = imsr.result;
if (timer != null) {
timer.stop();
log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
}
ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
for (TKeyValue kv : scanResult.results) {
entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
}
if (!entries.isEmpty())
receiver.receive(entries);
if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
timeoutTracker.madeProgress();
trackScanning(failures, unscanned, scanResult);
AtomicLong nextOpid = new AtomicLong();
while (scanResult.more) {
timeoutTracker.check();
if (timer != null) {
log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
timer.reset().start();
}
scanResult = client.continueMultiScan(TraceUtil.traceInfo(), imsr.scanID);
if (timer != null) {
timer.stop();
log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
}
entries = new ArrayList<>(scanResult.results.size());
for (TKeyValue kv : scanResult.results) {
entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
}
if (!entries.isEmpty())
receiver.receive(entries);
if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
timeoutTracker.madeProgress();
trackScanning(failures, unscanned, scanResult);
}
client.closeMultiScan(TraceUtil.traceInfo(), imsr.scanID);
} finally {
ThriftUtil.returnClient(client, context);
}
} catch (TTransportException e) {
log.debug("Server : {} msg : {}", server, e.getMessage());
timeoutTracker.errorOccured();
throw new IOException(e);
} catch (ThriftSecurityException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (TApplicationException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new AccumuloServerException(server, e);
} catch (NoSuchScanIDException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new IOException(e);
} catch (TSampleNotPresentException e) {
log.debug("Server : " + server + " msg : " + e.getMessage(), e);
String tableInfo = "?";
if (e.getExtent() != null) {
TableId tableId = KeyExtent.fromThrift(e.getExtent()).tableId();
tableInfo = context.getPrintableTableInfoFromId(tableId);
}
String message = "Table " + tableInfo + " does not have sampling configured or built";
throw new SampleNotPresentException(message, e);
} catch (TException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
timeoutTracker.errorOccured();
throw new IOException(e);
}
}
use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.
the class TabletServerBatchReaderIterator method trackScanning.
static void trackScanning(Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, MultiScanResult scanResult) {
// translate returned failures, remove them from unscanned, and add them to failures
// @formatter:off
Map<KeyExtent, List<Range>> retFailures = scanResult.failures.entrySet().stream().collect(Collectors.toMap((entry) -> KeyExtent.fromThrift(entry.getKey()), (entry) -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
// @formatter:on
unscanned.keySet().removeAll(retFailures.keySet());
failures.putAll(retFailures);
// translate full scans and remove them from unscanned
Set<KeyExtent> fullScans = scanResult.fullScans.stream().map(KeyExtent::fromThrift).collect(Collectors.toSet());
unscanned.keySet().removeAll(fullScans);
// remove partial scan from unscanned
if (scanResult.partScan != null) {
KeyExtent ke = KeyExtent.fromThrift(scanResult.partScan);
Key nextKey = new Key(scanResult.partNextKey);
ListIterator<Range> iterator = unscanned.get(ke).listIterator();
while (iterator.hasNext()) {
Range range = iterator.next();
if (range.afterEndKey(nextKey) || (nextKey.equals(range.getEndKey()) && scanResult.partNextKeyInclusive != range.isEndKeyInclusive())) {
iterator.remove();
} else if (range.contains(nextKey)) {
iterator.remove();
Range partRange = new Range(nextKey, scanResult.partNextKeyInclusive, range.getEndKey(), range.isEndKeyInclusive());
iterator.add(partRange);
}
}
}
}
use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.
the class VerifyTabletAssignments method checkTabletServer.
private static void checkTabletServer(ClientContext context, Entry<HostAndPort, List<KeyExtent>> entry, HashSet<KeyExtent> failures) throws ThriftSecurityException, TException, NoSuchScanIDException {
TabletClientService.Iface client = ThriftUtil.getTServerClient(entry.getKey(), context);
Map<TKeyExtent, List<TRange>> batch = new TreeMap<>();
for (KeyExtent keyExtent : entry.getValue()) {
Text row = keyExtent.endRow();
Text row2 = null;
if (row == null) {
row = keyExtent.prevEndRow();
if (row != null) {
row = new Text(row);
row.append(new byte[] { 'a' }, 0, 1);
} else {
row = new Text("1234567890");
}
row2 = new Text(row);
row2.append(new byte[] { '!' }, 0, 1);
} else {
row = new Text(row);
row2 = new Text(row);
row.getBytes()[row.getLength() - 1] = (byte) (row.getBytes()[row.getLength() - 1] - 1);
}
Range r = new Range(row, true, row2, false);
batch.put(keyExtent.toThrift(), Collections.singletonList(r.toThrift()));
}
TInfo tinfo = TraceUtil.traceInfo();
Map<String, Map<String, String>> emptyMapSMapSS = Collections.emptyMap();
List<IterInfo> emptyListIterInfo = Collections.emptyList();
List<TColumn> emptyListColumn = Collections.emptyList();
InitialMultiScan is = client.startMultiScan(tinfo, context.rpcCreds(), batch, emptyListColumn, emptyListIterInfo, emptyMapSMapSS, Authorizations.EMPTY.getAuthorizationsBB(), false, null, 0L, null, null);
if (is.result.more) {
MultiScanResult result = client.continueMultiScan(tinfo, is.scanID);
checkFailures(entry.getKey(), failures, result);
while (result.more) {
result = client.continueMultiScan(tinfo, is.scanID);
checkFailures(entry.getKey(), failures, result);
}
}
client.closeMultiScan(tinfo, is.scanID);
ThriftUtil.returnClient((TServiceClient) client, context);
}
use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.
the class ThriftClientHandler method startMultiScan.
@Override
public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent, List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws ThriftSecurityException, TSampleNotPresentException {
// find all of the tables that need to be scanned
final HashSet<TableId> tables = new HashSet<>();
for (TKeyExtent keyExtent : tbatch.keySet()) {
tables.add(TableId.of(new String(keyExtent.getTable(), UTF_8)));
}
if (tables.size() != 1) {
throw new IllegalArgumentException("Cannot batch scan over multiple tables");
}
// check if user has permission to the tables
for (TableId tableId : tables) {
NamespaceId namespaceId = getNamespaceId(credentials, tableId);
if (!security.canScan(credentials, tableId, namespaceId, tbatch, tcolumns, ssiList, ssio, authorizations)) {
throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
}
}
try {
if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
}
} catch (ThriftSecurityException tse) {
log.error("{} is not authorized", credentials.getPrincipal(), tse);
throw tse;
}
// @formatter:off
Map<KeyExtent, List<Range>> batch = tbatch.entrySet().stream().collect(Collectors.toMap(entry -> KeyExtent.fromThrift(entry.getKey()), entry -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
// @formatter:on
// This is used to determine which thread pool to use
KeyExtent threadPoolExtent = batch.keySet().iterator().next();
if (waitForWrites) {
writeTracker.waitForWrites(TabletType.type(batch.keySet()));
}
Set<Column> columnSet = tcolumns.isEmpty() ? Collections.emptySet() : new HashSet<>(Collections2.transform(tcolumns, Column::new));
ScanParameters scanParams = new ScanParameters(-1, new Authorizations(authorizations), columnSet, ssiList, ssio, false, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
final MultiScanSession mss = new MultiScanSession(credentials, threadPoolExtent, batch, scanParams, executionHints);
mss.numTablets = batch.size();
for (List<Range> ranges : batch.values()) {
mss.numRanges += ranges.size();
}
long sid = server.sessionManager.createSession(mss, true);
MultiScanResult result;
try {
result = continueMultiScan(sid, mss);
} finally {
server.sessionManager.unreserveSession(sid);
}
return new InitialMultiScan(sid, result);
}
use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.
the class LookupTask method run.
@Override
public void run() {
MultiScanSession session = (MultiScanSession) server.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
try {
if (isCancelled() || session == null)
return;
TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
runState.set(ScanRunState.RUNNING);
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
long bytesAdded = 0;
long maxScanTime = 4000;
long startTime = System.currentTimeMillis();
List<KVEntry> results = new ArrayList<>();
Map<KeyExtent, List<Range>> failures = new HashMap<>();
List<KeyExtent> fullScans = new ArrayList<>();
KeyExtent partScan = null;
Key partNextKey = null;
boolean partNextKeyInclusive = false;
Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
// check the time so that the read ahead thread is not monopolized
while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
Entry<KeyExtent, List<Range>> entry = iter.next();
iter.remove();
// check that tablet server is serving requested tablet
Tablet tablet = server.getOnlineTablet(entry.getKey());
if (tablet == null) {
failures.put(entry.getKey(), entry.getValue());
continue;
}
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
LookupResult lookupResult;
try {
// canceled
if (isCancelled())
interruptFlag.set(true);
lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
// if the tablet was closed it it possible that the
// interrupt flag was set.... do not want it set for
// the next
// lookup
interruptFlag.set(false);
} catch (IOException e) {
log.warn("lookup failed for tablet " + entry.getKey(), e);
throw new RuntimeException(e);
}
bytesAdded += lookupResult.bytesAdded;
if (lookupResult.unfinishedRanges.isEmpty()) {
fullScans.add(entry.getKey());
} else {
if (lookupResult.closed) {
failures.put(entry.getKey(), lookupResult.unfinishedRanges);
} else {
session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
partScan = entry.getKey();
partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
}
}
}
long finishTime = System.currentTimeMillis();
session.totalLookupTime += (finishTime - startTime);
session.numEntries += results.size();
// convert everything to thrift before adding result
List<TKeyValue> retResults = new ArrayList<>();
for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
// @formatter:off
Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
// @formatter:on
List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
TKeyExtent retPartScan = null;
TKey retPartNextKey = null;
if (partScan != null) {
retPartScan = partScan.toThrift();
retPartNextKey = partNextKey.toThrift();
}
// add results to queue
addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
} catch (IterationInterruptedException iie) {
if (!isCancelled()) {
log.warn("Iteration interrupted, when scan not cancelled", iie);
addResult(iie);
}
} catch (SampleNotPresentException e) {
addResult(e);
} catch (Exception e) {
log.warn("exception while doing multi-scan ", e);
addResult(e);
} finally {
Thread.currentThread().setName(oldThreadName);
runState.set(ScanRunState.FINISHED);
}
}
Aggregations