use of org.apache.accumulo.core.dataImpl.thrift.TKey in project accumulo by apache.
the class Key method decompress.
/**
* Decompresses a list of key/value pairs received from thrift. Decompression occurs in place, in
* the list.
*
* @param param
* list of Thrift key/value pairs
*/
public static void decompress(List<TKeyValue> param) {
for (int i = 1; i < param.size(); i++) {
TKey prevKey = param.get(i - 1).key;
TKey key = param.get(i).key;
if (key.row == null) {
key.row = prevKey.row;
}
if (key.colFamily == null) {
key.colFamily = prevKey.colFamily;
}
if (key.colQualifier == null) {
key.colQualifier = prevKey.colQualifier;
}
if (key.colVisibility == null) {
key.colVisibility = prevKey.colVisibility;
}
}
}
use of org.apache.accumulo.core.dataImpl.thrift.TKey in project accumulo by apache.
the class KeyTest method testThrift.
@Test
public void testThrift() {
Key k = new Key("r1", "cf2", "cq2", "cv");
TKey tk = k.toThrift();
Key k2 = new Key(tk);
assertEquals(k, k2);
}
use of org.apache.accumulo.core.dataImpl.thrift.TKey in project accumulo by apache.
the class Key method compress.
/**
* Compresses a list of key/value pairs before sending them via thrift.
*
* @param param
* list of key/value pairs
* @return list of Thrift key/value pairs
*/
public static List<TKeyValue> compress(List<? extends KeyValue> param) {
List<TKeyValue> tkvl = Arrays.asList(new TKeyValue[param.size()]);
if (!param.isEmpty())
tkvl.set(0, new TKeyValue(param.get(0).getKey().toThrift(), ByteBuffer.wrap(param.get(0).getValue().get())));
for (int i = param.size() - 1; i > 0; i--) {
Key prevKey = param.get(i - 1).getKey();
KeyValue kv = param.get(i);
Key key = kv.getKey();
TKey newKey = null;
if (isEqual(prevKey.row, key.row)) {
newKey = key.toThrift();
newKey.row = null;
}
if (isEqual(prevKey.colFamily, key.colFamily)) {
if (newKey == null)
newKey = key.toThrift();
newKey.colFamily = null;
}
if (isEqual(prevKey.colQualifier, key.colQualifier)) {
if (newKey == null)
newKey = key.toThrift();
newKey.colQualifier = null;
}
if (isEqual(prevKey.colVisibility, key.colVisibility)) {
if (newKey == null)
newKey = key.toThrift();
newKey.colVisibility = null;
}
if (newKey == null)
newKey = key.toThrift();
tkvl.set(i, new TKeyValue(newKey, ByteBuffer.wrap(kv.getValue().get())));
}
return tkvl;
}
use of org.apache.accumulo.core.dataImpl.thrift.TKey in project accumulo by apache.
the class KeyTest method testThrift_Invalid.
@Test
public void testThrift_Invalid() {
Key k = new Key("r1", "cf2", "cq2", "cv");
TKey tk = k.toThrift();
tk.setRow((byte[]) null);
assertThrows(IllegalArgumentException.class, () -> new Key(tk));
}
use of org.apache.accumulo.core.dataImpl.thrift.TKey in project accumulo by apache.
the class LookupTask method run.
@Override
public void run() {
MultiScanSession session = (MultiScanSession) server.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
try {
if (isCancelled() || session == null)
return;
TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
runState.set(ScanRunState.RUNNING);
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
long bytesAdded = 0;
long maxScanTime = 4000;
long startTime = System.currentTimeMillis();
List<KVEntry> results = new ArrayList<>();
Map<KeyExtent, List<Range>> failures = new HashMap<>();
List<KeyExtent> fullScans = new ArrayList<>();
KeyExtent partScan = null;
Key partNextKey = null;
boolean partNextKeyInclusive = false;
Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
// check the time so that the read ahead thread is not monopolized
while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
Entry<KeyExtent, List<Range>> entry = iter.next();
iter.remove();
// check that tablet server is serving requested tablet
Tablet tablet = server.getOnlineTablet(entry.getKey());
if (tablet == null) {
failures.put(entry.getKey(), entry.getValue());
continue;
}
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
LookupResult lookupResult;
try {
// canceled
if (isCancelled())
interruptFlag.set(true);
lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
// if the tablet was closed it it possible that the
// interrupt flag was set.... do not want it set for
// the next
// lookup
interruptFlag.set(false);
} catch (IOException e) {
log.warn("lookup failed for tablet " + entry.getKey(), e);
throw new RuntimeException(e);
}
bytesAdded += lookupResult.bytesAdded;
if (lookupResult.unfinishedRanges.isEmpty()) {
fullScans.add(entry.getKey());
} else {
if (lookupResult.closed) {
failures.put(entry.getKey(), lookupResult.unfinishedRanges);
} else {
session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
partScan = entry.getKey();
partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
}
}
}
long finishTime = System.currentTimeMillis();
session.totalLookupTime += (finishTime - startTime);
session.numEntries += results.size();
// convert everything to thrift before adding result
List<TKeyValue> retResults = new ArrayList<>();
for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
// @formatter:off
Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
// @formatter:on
List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
TKeyExtent retPartScan = null;
TKey retPartNextKey = null;
if (partScan != null) {
retPartScan = partScan.toThrift();
retPartNextKey = partNextKey.toThrift();
}
// add results to queue
addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
} catch (IterationInterruptedException iie) {
if (!isCancelled()) {
log.warn("Iteration interrupted, when scan not cancelled", iie);
addResult(iie);
}
} catch (SampleNotPresentException e) {
addResult(e);
} catch (Exception e) {
log.warn("exception while doing multi-scan ", e);
addResult(e);
} finally {
Thread.currentThread().setName(oldThreadName);
runState.set(ScanRunState.FINISHED);
}
}
Aggregations