Examples with TKeyValue org.apache.accumulo.core.data.thrift.TKeyValue used on opensource projects

Example 1 with TKeyValue

use of org.apache.accumulo.core.data.thrift.TKeyValue in project accumulo by apache.

the class TabletServerBatchReaderIterator method doLookup.

static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
    if (requested.size() == 0) {
        return;
    }
    // copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
    for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
        ArrayList<Range> ranges = new ArrayList<>();
        for (Range range : entry.getValue()) {
            ranges.add(new Range(range));
        }
        unscanned.put(new KeyExtent(entry.getKey()), ranges);
    }
    timeoutTracker.startingScan();
    TTransport transport = null;
    try {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        final TabletClientService.Client client;
        if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
            client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
        else
            client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            OpTimer timer = null;
            if (log.isTraceEnabled()) {
                log.trace("tid={} Starting multi scan, tserver={}  #tablets={}  #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(requested.keySet());
            boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
            Map<TKeyExtent, List<TRange>> thriftTabletRanges = Translator.translate(requested, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
            InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), context.rpcCreds(), thriftTabletRanges, Translator.translate(columns, Translators.CT), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext);
            if (waitForWrites)
                ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
            MultiScanResult scanResult = imsr.result;
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
            }
            ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
            for (TKeyValue kv : scanResult.results) {
                entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
            }
            if (entries.size() > 0)
                receiver.receive(entries);
            if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                timeoutTracker.madeProgress();
            trackScanning(failures, unscanned, scanResult);
            AtomicLong nextOpid = new AtomicLong();
            while (scanResult.more) {
                timeoutTracker.check();
                if (timer != null) {
                    log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
                    timer.reset().start();
                }
                scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
                }
                entries = new ArrayList<>(scanResult.results.size());
                for (TKeyValue kv : scanResult.results) {
                    entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
                }
                if (entries.size() > 0)
                    receiver.receive(entries);
                if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                    timeoutTracker.madeProgress();
                trackScanning(failures, unscanned, scanResult);
            }
            client.closeMultiScan(Tracer.traceInfo(), imsr.scanID);
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TTransportException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage());
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } catch (ThriftSecurityException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TApplicationException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloServerException(server, e);
    } catch (NoSuchScanIDException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new IOException(e);
    } catch (TSampleNotPresentException e) {
        log.debug("Server : " + server + " msg : " + e.getMessage(), e);
        String tableInfo = "?";
        if (e.getExtent() != null) {
            Table.ID tableId = new KeyExtent(e.getExtent()).getTableId();
            tableInfo = Tables.getPrintableTableInfoFromId(context.getInstance(), tableId);
        }
        String message = "Table " + tableInfo + " does not have sampling configured or built";
        throw new SampleNotPresentException(message, e);
    } catch (TException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } finally {
        ThriftTransportPool.getInstance().returnTransport(transport);
    }
}

Example 2 with TKeyValue

use of org.apache.accumulo.core.data.thrift.TKeyValue in project accumulo by apache.

the class KeyTest method testCompressDecompress.

@Test
public void testCompressDecompress() {
    List<KeyValue> kvs = new ArrayList<>();
    kvs.add(new KeyValue(new Key(), new byte[] {}));
    kvs.add(new KeyValue(new Key("r"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r", "cf"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r2", "cf"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r", "cf", "cq"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r2", "cf2", "cq"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r", "cf", "cq", "cv"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r2", "cf2", "cq2", "cv"), new byte[] {}));
    kvs.add(new KeyValue(new Key("r2", "cf2", "cq2", "cv"), new byte[] {}));
    kvs.add(new KeyValue(new Key(), new byte[] {}));
    List<TKeyValue> tkvs = Key.compress(kvs);
    Key.decompress(tkvs);
    assertEquals(kvs.size(), tkvs.size());
    Iterator<KeyValue> kvi = kvs.iterator();
    Iterator<TKeyValue> tkvi = tkvs.iterator();
    while (kvi.hasNext()) {
        KeyValue kv = kvi.next();
        TKeyValue tkv = tkvi.next();
        assertEquals(kv.getKey(), new Key(tkv.getKey()));
    }
}

Example 3 with TKeyValue

use of org.apache.accumulo.core.data.thrift.TKeyValue in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList, session.ssio, interruptFlag, session.samplerConfig, session.batchTimeOut, session.context);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.size() > 0) {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            } else {
                fullScans.add(entry.getKey());
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        Map<TKeyExtent, List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
        List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Throwable e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}

Example 4 with TKeyValue

use of org.apache.accumulo.core.data.thrift.TKeyValue in project accumulo by apache.

the class Key method compress.

/**
 * Compresses a list of key/value pairs before sending them via thrift.
 *
 * @param param
 *          list of key/value pairs
 * @return list of Thrift key/value pairs
 */
public static List<TKeyValue> compress(List<? extends KeyValue> param) {
    List<TKeyValue> tkvl = Arrays.asList(new TKeyValue[param.size()]);
    if (param.size() > 0)
        tkvl.set(0, new TKeyValue(param.get(0).getKey().toThrift(), ByteBuffer.wrap(param.get(0).getValue().get())));
    for (int i = param.size() - 1; i > 0; i--) {
        Key prevKey = param.get(i - 1).getKey();
        KeyValue kv = param.get(i);
        Key key = kv.getKey();
        TKey newKey = null;
        if (isEqual(prevKey.row, key.row)) {
            newKey = key.toThrift();
            newKey.row = null;
        }
        if (isEqual(prevKey.colFamily, key.colFamily)) {
            if (newKey == null)
                newKey = key.toThrift();
            newKey.colFamily = null;
        }
        if (isEqual(prevKey.colQualifier, key.colQualifier)) {
            if (newKey == null)
                newKey = key.toThrift();
            newKey.colQualifier = null;
        }
        if (isEqual(prevKey.colVisibility, key.colVisibility)) {
            if (newKey == null)
                newKey = key.toThrift();
            newKey.colVisibility = null;
        }
        if (newKey == null)
            newKey = key.toThrift();
        tkvl.set(i, new TKeyValue(newKey, ByteBuffer.wrap(kv.getValue().get())));
    }
    return tkvl;
}

Example 5 with TKeyValue

use of org.apache.accumulo.core.data.thrift.TKeyValue in project accumulo by apache.

the class ThriftScanner method getBatchFromServer.

public static boolean getBatchFromServer(ClientContext context, Range range, KeyExtent extent, String server, SortedMap<Key, Value> results, SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList, Map<String, Map<String, String>> serverSideIteratorOptions, int size, Authorizations authorizations, boolean retry, long batchTimeOut, String classLoaderContext) throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
    if (server == null)
        throw new AccumuloException(new IOException());
    final HostAndPort parsedServer = HostAndPort.fromString(server);
    try {
        TInfo tinfo = Tracer.traceInfo();
        TabletClientService.Client client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            // not reading whole rows (or stopping on row boundries) so there is no need to enable isolation below
            ScanState scanState = new ScanState(context, extent.getTableId(), authorizations, range, fetchedColumns, size, serverSideIteratorList, serverSideIteratorOptions, false, Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD, null, batchTimeOut, classLoaderContext);
            TabletType ttype = TabletType.type(extent);
            boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);
            InitialScan isr = client.startScan(tinfo, scanState.context.rpcCreds(), extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, null, scanState.batchTimeOut, classLoaderContext);
            if (waitForWrites)
                serversWaitedForWrites.get(ttype).add(server);
            Key.decompress(isr.result.results);
            for (TKeyValue kv : isr.result.results) results.put(new Key(kv.key), new Value(kv.value));
            client.closeScan(tinfo, isr.scanID);
            return isr.result.more;
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TApplicationException tae) {
        throw new AccumuloServerException(server, tae);
    } catch (TooManyFilesException e) {
        log.debug("Tablet ({}) has too many files {} : {}", extent, server, e.getMessage());
    } catch (ThriftSecurityException e) {
        log.warn("Security Violation in scan request to {}: {}", server, e.getMessage());
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TException e) {
        log.debug("Error getting transport to {}: {}", server, e.getMessage());
    }
    throw new AccumuloException("getBatchFromServer: failed");
}