use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.
the class TabletServerBatchReaderIterator method doLookup.
static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
if (requested.isEmpty()) {
return;
}
// copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
ArrayList<Range> ranges = new ArrayList<>();
for (Range range : entry.getValue()) {
ranges.add(new Range(range));
}
unscanned.put(KeyExtent.copyOf(entry.getKey()), ranges);
}
timeoutTracker.startingScan();
try {
final HostAndPort parsedServer = HostAndPort.fromString(server);
final TabletClientService.Client client;
if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
else
client = ThriftUtil.getTServerClient(parsedServer, context);
try {
OpTimer timer = null;
if (log.isTraceEnabled()) {
log.trace("tid={} Starting multi scan, tserver={} #tablets={} #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
timer = new OpTimer().start();
}
TabletType ttype = TabletType.type(requested.keySet());
boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
// @formatter:off
Map<TKeyExtent, List<TRange>> thriftTabletRanges = requested.entrySet().stream().collect(Collectors.toMap((entry) -> entry.getKey().toThrift(), (entry) -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
// @formatter:on
Map<String, String> execHints = options.executionHints.isEmpty() ? null : options.executionHints;
InitialMultiScan imsr = client.startMultiScan(TraceUtil.traceInfo(), context.rpcCreds(), thriftTabletRanges, columns.stream().map(Column::toThrift).collect(Collectors.toList()), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext, execHints);
if (waitForWrites)
ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
MultiScanResult scanResult = imsr.result;
if (timer != null) {
timer.stop();
log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
}
ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
for (TKeyValue kv : scanResult.results) {
entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
}
if (!entries.isEmpty())
receiver.receive(entries);
if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
timeoutTracker.madeProgress();
trackScanning(failures, unscanned, scanResult);
AtomicLong nextOpid = new AtomicLong();
while (scanResult.more) {
timeoutTracker.check();
if (timer != null) {
log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
timer.reset().start();
}
scanResult = client.continueMultiScan(TraceUtil.traceInfo(), imsr.scanID);
if (timer != null) {
timer.stop();
log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
}
entries = new ArrayList<>(scanResult.results.size());
for (TKeyValue kv : scanResult.results) {
entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
}
if (!entries.isEmpty())
receiver.receive(entries);
if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
timeoutTracker.madeProgress();
trackScanning(failures, unscanned, scanResult);
}
client.closeMultiScan(TraceUtil.traceInfo(), imsr.scanID);
} finally {
ThriftUtil.returnClient(client, context);
}
} catch (TTransportException e) {
log.debug("Server : {} msg : {}", server, e.getMessage());
timeoutTracker.errorOccured();
throw new IOException(e);
} catch (ThriftSecurityException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (TApplicationException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new AccumuloServerException(server, e);
} catch (NoSuchScanIDException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
throw new IOException(e);
} catch (TSampleNotPresentException e) {
log.debug("Server : " + server + " msg : " + e.getMessage(), e);
String tableInfo = "?";
if (e.getExtent() != null) {
TableId tableId = KeyExtent.fromThrift(e.getExtent()).tableId();
tableInfo = context.getPrintableTableInfoFromId(tableId);
}
String message = "Table " + tableInfo + " does not have sampling configured or built";
throw new SampleNotPresentException(message, e);
} catch (TException e) {
log.debug("Server : {} msg : {}", server, e.getMessage(), e);
timeoutTracker.errorOccured();
throw new IOException(e);
}
}
use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.
the class KeyExtent method fromThrift.
/**
* Create a KeyExtent from its Thrift form.
*
* @param tke
* the KeyExtent in its Thrift object form
*/
public static KeyExtent fromThrift(TKeyExtent tke) {
TableId tableId = TableId.of(new String(ByteBufferUtil.toBytes(tke.table), UTF_8));
Text endRow = tke.endRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.endRow));
Text prevEndRow = tke.prevEndRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.prevEndRow));
return new KeyExtent(tableId, endRow, prevEndRow);
}
use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.
the class ConditionalWriterImpl method sendToServer.
private void sendToServer(HostAndPort location, TabletServerMutations<QCMutation> mutations) {
TabletClientService.Iface client = null;
TInfo tinfo = TraceUtil.traceInfo();
Map<Long, CMK> cmidToCm = new HashMap<>();
MutableLong cmid = new MutableLong(0);
SessionID sessionId = null;
try {
Map<TKeyExtent, List<TConditionalMutation>> tmutations = new HashMap<>();
CompressedIterators compressedIters = new CompressedIterators();
convertMutations(mutations, cmidToCm, cmid, tmutations, compressedIters);
// getClient() call must come after converMutations in case it throws a TException
client = getClient(location);
List<TCMResult> tresults = null;
while (tresults == null) {
try {
sessionId = reserveSessionID(location, client, tinfo);
tresults = client.conditionalUpdate(tinfo, sessionId.sessionID, tmutations, compressedIters.getSymbolTable());
} catch (NoSuchScanIDException nssie) {
sessionId = null;
invalidateSessionID(location);
}
}
HashSet<KeyExtent> extentsToInvalidate = new HashSet<>();
ArrayList<QCMutation> ignored = new ArrayList<>();
for (TCMResult tcmResult : tresults) {
if (tcmResult.status == TCMStatus.IGNORED) {
CMK cmk = cmidToCm.get(tcmResult.cmid);
ignored.add(cmk.cm);
extentsToInvalidate.add(cmk.ke);
} else {
QCMutation qcm = cmidToCm.get(tcmResult.cmid).cm;
qcm.queueResult(new Result(fromThrift(tcmResult.status), qcm, location.toString()));
}
}
for (KeyExtent ke : extentsToInvalidate) {
locator.invalidateCache(ke);
}
queueRetry(ignored, location);
} catch (ThriftSecurityException tse) {
AccumuloSecurityException ase = new AccumuloSecurityException(context.getCredentials().getPrincipal(), tse.getCode(), context.getPrintableTableInfoFromId(tableId), tse);
queueException(location, cmidToCm, ase);
} catch (TApplicationException tae) {
queueException(location, cmidToCm, new AccumuloServerException(location.toString(), tae));
} catch (TException e) {
locator.invalidateCache(context, location.toString());
invalidateSession(location, cmidToCm, sessionId);
} catch (Exception e) {
queueException(location, cmidToCm, e);
} finally {
if (sessionId != null)
unreserveSessionID(location);
ThriftUtil.returnClient((TServiceClient) client, context);
}
}
use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.
the class CompactorTest method testCompactionSucceeds.
@Test
public void testCompactionSucceeds() throws Exception {
UUID uuid = UUID.randomUUID();
Supplier<UUID> supplier = () -> uuid;
ExternalCompactionId eci = ExternalCompactionId.generate(supplier.get());
PowerMock.resetAll();
PowerMock.suppress(PowerMock.methods(Halt.class, "halt"));
PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
HostAndPort address = HostAndPort.fromString("localhost:10240");
EasyMock.expect(client.getAddress()).andReturn(address);
TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
TKeyExtent extent = PowerMock.createNiceMock(TKeyExtent.class);
EasyMock.expect(job.isSetExternalCompactionId()).andReturn(true).anyTimes();
EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
EasyMock.expect(job.getExtent()).andReturn(extent).anyTimes();
EasyMock.expect(extent.getTable()).andReturn("testTable".getBytes()).anyTimes();
AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
EasyMock.expect(conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)).andReturn(86400000L);
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
EasyMock.expect(context.getConfiguration()).andReturn(conf);
ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
EasyMock.expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
EasyMock.expect(zrw.getZooKeeper()).andReturn(zk).anyTimes();
VolumeManagerImpl vm = PowerMock.createNiceMock(VolumeManagerImpl.class);
EasyMock.expect(context.getVolumeManager()).andReturn(vm);
vm.close();
PowerMock.replayAll();
SuccessfulCompactor c = new SuccessfulCompactor(supplier, client, job, conf, context, eci);
c.run();
PowerMock.verifyAll();
c.close();
assertTrue(c.isCompletedCalled());
assertFalse(c.isFailedCalled());
}
use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.
the class BulkImporter method assignMapFiles.
private List<KeyExtent> assignMapFiles(ClientContext context, HostAndPort location, Map<KeyExtent, List<PathSize>> assignmentsPerTablet) throws AccumuloException, AccumuloSecurityException {
try {
long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT);
TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
try {
HashMap<KeyExtent, Map<String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files = new HashMap<>();
for (Entry<KeyExtent, List<PathSize>> entry : assignmentsPerTablet.entrySet()) {
HashMap<String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> tabletFiles = new HashMap<>();
files.put(entry.getKey(), tabletFiles);
for (PathSize pathSize : entry.getValue()) {
org.apache.accumulo.core.dataImpl.thrift.MapFileInfo mfi = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo(pathSize.estSize);
tabletFiles.put(pathSize.path.toString(), mfi);
}
}
log.debug("Asking {} to bulk load {}", location, files);
List<TKeyExtent> failures = client.bulkImport(TraceUtil.traceInfo(), context.rpcCreds(), tid, files.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), Entry::getValue)), setTime);
return failures.stream().map(KeyExtent::fromThrift).collect(Collectors.toList());
} finally {
ThriftUtil.returnClient((TServiceClient) client, context);
}
} catch (ThriftSecurityException e) {
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (Exception t) {
log.error("Encountered unknown exception in assignMapFiles.", t);
throw new AccumuloException(t);
}
}
Aggregations