Search in sources :

Example 1 with InitialMultiScan

use of org.apache.accumulo.core.data.thrift.InitialMultiScan in project accumulo by apache.

the class TabletServerBatchReaderIterator method doLookup.

static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
    if (requested.size() == 0) {
        return;
    }
    // copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
    for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
        ArrayList<Range> ranges = new ArrayList<>();
        for (Range range : entry.getValue()) {
            ranges.add(new Range(range));
        }
        unscanned.put(new KeyExtent(entry.getKey()), ranges);
    }
    timeoutTracker.startingScan();
    TTransport transport = null;
    try {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        final TabletClientService.Client client;
        if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
            client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
        else
            client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            OpTimer timer = null;
            if (log.isTraceEnabled()) {
                log.trace("tid={} Starting multi scan, tserver={}  #tablets={}  #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(requested.keySet());
            boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
            Map<TKeyExtent, List<TRange>> thriftTabletRanges = Translator.translate(requested, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
            InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), context.rpcCreds(), thriftTabletRanges, Translator.translate(columns, Translators.CT), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext);
            if (waitForWrites)
                ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
            MultiScanResult scanResult = imsr.result;
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
            }
            ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
            for (TKeyValue kv : scanResult.results) {
                entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
            }
            if (entries.size() > 0)
                receiver.receive(entries);
            if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                timeoutTracker.madeProgress();
            trackScanning(failures, unscanned, scanResult);
            AtomicLong nextOpid = new AtomicLong();
            while (scanResult.more) {
                timeoutTracker.check();
                if (timer != null) {
                    log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
                    timer.reset().start();
                }
                scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
                }
                entries = new ArrayList<>(scanResult.results.size());
                for (TKeyValue kv : scanResult.results) {
                    entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
                }
                if (entries.size() > 0)
                    receiver.receive(entries);
                if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                    timeoutTracker.madeProgress();
                trackScanning(failures, unscanned, scanResult);
            }
            client.closeMultiScan(Tracer.traceInfo(), imsr.scanID);
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TTransportException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage());
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } catch (ThriftSecurityException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TApplicationException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloServerException(server, e);
    } catch (NoSuchScanIDException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new IOException(e);
    } catch (TSampleNotPresentException e) {
        log.debug("Server : " + server + " msg : " + e.getMessage(), e);
        String tableInfo = "?";
        if (e.getExtent() != null) {
            Table.ID tableId = new KeyExtent(e.getExtent()).getTableId();
            tableInfo = Tables.getPrintableTableInfoFromId(context.getInstance(), tableId);
        }
        String message = "Table " + tableInfo + " does not have sampling configured or built";
        throw new SampleNotPresentException(message, e);
    } catch (TException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } finally {
        ThriftTransportPool.getInstance().returnTransport(transport);
    }
}
Also used : TException(org.apache.thrift.TException) ArrayList(java.util.ArrayList) TTransportException(org.apache.thrift.transport.TTransportException) TKeyExtent(org.apache.accumulo.core.data.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) HostAndPort(org.apache.accumulo.core.util.HostAndPort) Entry(java.util.Map.Entry) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) MultiScanResult(org.apache.accumulo.core.data.thrift.MultiScanResult) List(java.util.List) ArrayList(java.util.ArrayList) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) TKeyValue(org.apache.accumulo.core.data.thrift.TKeyValue) InitialMultiScan(org.apache.accumulo.core.data.thrift.InitialMultiScan) TKeyExtent(org.apache.accumulo.core.data.thrift.TKeyExtent) IOException(java.io.IOException) TRange(org.apache.accumulo.core.data.thrift.TRange) Range(org.apache.accumulo.core.data.Range) ThriftSecurityException(org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) TApplicationException(org.apache.thrift.TApplicationException) AtomicLong(java.util.concurrent.atomic.AtomicLong) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) OpTimer(org.apache.accumulo.core.util.OpTimer) TKeyValue(org.apache.accumulo.core.data.thrift.TKeyValue) Value(org.apache.accumulo.core.data.Value) TTransport(org.apache.thrift.transport.TTransport) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Key(org.apache.accumulo.core.data.Key)

Example 2 with InitialMultiScan

use of org.apache.accumulo.core.data.thrift.InitialMultiScan in project accumulo by apache.

the class VerifyTabletAssignments method checkTabletServer.

private static void checkTabletServer(ClientContext context, Entry<HostAndPort, List<KeyExtent>> entry, HashSet<KeyExtent> failures) throws ThriftSecurityException, TException, NoSuchScanIDException {
    TabletClientService.Iface client = ThriftUtil.getTServerClient(entry.getKey(), context);
    Map<TKeyExtent, List<TRange>> batch = new TreeMap<>();
    for (KeyExtent keyExtent : entry.getValue()) {
        Text row = keyExtent.getEndRow();
        Text row2 = null;
        if (row == null) {
            row = keyExtent.getPrevEndRow();
            if (row != null) {
                row = new Text(row);
                row.append(new byte[] { 'a' }, 0, 1);
            } else {
                row = new Text("1234567890");
            }
            row2 = new Text(row);
            row2.append(new byte[] { '!' }, 0, 1);
        } else {
            row = new Text(row);
            row2 = new Text(row);
            row.getBytes()[row.getLength() - 1] = (byte) (row.getBytes()[row.getLength() - 1] - 1);
        }
        Range r = new Range(row, true, row2, false);
        batch.put(keyExtent.toThrift(), Collections.singletonList(r.toThrift()));
    }
    TInfo tinfo = Tracer.traceInfo();
    Map<String, Map<String, String>> emptyMapSMapSS = Collections.emptyMap();
    List<IterInfo> emptyListIterInfo = Collections.emptyList();
    List<TColumn> emptyListColumn = Collections.emptyList();
    InitialMultiScan is = client.startMultiScan(tinfo, context.rpcCreds(), batch, emptyListColumn, emptyListIterInfo, emptyMapSMapSS, Authorizations.EMPTY.getAuthorizationsBB(), false, null, 0L, null);
    if (is.result.more) {
        MultiScanResult result = client.continueMultiScan(tinfo, is.scanID);
        checkFailures(entry.getKey(), failures, result);
        while (result.more) {
            result = client.continueMultiScan(tinfo, is.scanID);
            checkFailures(entry.getKey(), failures, result);
        }
    }
    client.closeMultiScan(tinfo, is.scanID);
    ThriftUtil.returnClient((TServiceClient) client);
}
Also used : TInfo(org.apache.accumulo.core.trace.thrift.TInfo) TColumn(org.apache.accumulo.core.data.thrift.TColumn) InitialMultiScan(org.apache.accumulo.core.data.thrift.InitialMultiScan) Text(org.apache.hadoop.io.Text) TKeyExtent(org.apache.accumulo.core.data.thrift.TKeyExtent) TreeMap(java.util.TreeMap) TRange(org.apache.accumulo.core.data.thrift.TRange) Range(org.apache.accumulo.core.data.Range) TKeyExtent(org.apache.accumulo.core.data.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) IterInfo(org.apache.accumulo.core.data.thrift.IterInfo) MultiScanResult(org.apache.accumulo.core.data.thrift.MultiScanResult) ArrayList(java.util.ArrayList) List(java.util.List) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Map(java.util.Map) TreeMap(java.util.TreeMap)

Aggregations

ArrayList (java.util.ArrayList)2 List (java.util.List)2 Range (org.apache.accumulo.core.data.Range)2 KeyExtent (org.apache.accumulo.core.data.impl.KeyExtent)2 InitialMultiScan (org.apache.accumulo.core.data.thrift.InitialMultiScan)2 MultiScanResult (org.apache.accumulo.core.data.thrift.MultiScanResult)2 TKeyExtent (org.apache.accumulo.core.data.thrift.TKeyExtent)2 TRange (org.apache.accumulo.core.data.thrift.TRange)2 TabletClientService (org.apache.accumulo.core.tabletserver.thrift.TabletClientService)2 IOException (java.io.IOException)1 SimpleImmutableEntry (java.util.AbstractMap.SimpleImmutableEntry)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 TreeMap (java.util.TreeMap)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 AccumuloSecurityException (org.apache.accumulo.core.client.AccumuloSecurityException)1 SampleNotPresentException (org.apache.accumulo.core.client.SampleNotPresentException)1 ThriftSecurityException (org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException)1 Key (org.apache.accumulo.core.data.Key)1 Value (org.apache.accumulo.core.data.Value)1