Search in sources :

Example 1 with MultiScanResult

use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.

the class TabletServerBatchReaderIterator method doLookup.

static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
    if (requested.isEmpty()) {
        return;
    }
    // copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
    for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
        ArrayList<Range> ranges = new ArrayList<>();
        for (Range range : entry.getValue()) {
            ranges.add(new Range(range));
        }
        unscanned.put(KeyExtent.copyOf(entry.getKey()), ranges);
    }
    timeoutTracker.startingScan();
    try {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        final TabletClientService.Client client;
        if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
            client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
        else
            client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            OpTimer timer = null;
            if (log.isTraceEnabled()) {
                log.trace("tid={} Starting multi scan, tserver={}  #tablets={}  #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(requested.keySet());
            boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
            // @formatter:off
            Map<TKeyExtent, List<TRange>> thriftTabletRanges = requested.entrySet().stream().collect(Collectors.toMap((entry) -> entry.getKey().toThrift(), (entry) -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
            // @formatter:on
            Map<String, String> execHints = options.executionHints.isEmpty() ? null : options.executionHints;
            InitialMultiScan imsr = client.startMultiScan(TraceUtil.traceInfo(), context.rpcCreds(), thriftTabletRanges, columns.stream().map(Column::toThrift).collect(Collectors.toList()), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext, execHints);
            if (waitForWrites)
                ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
            MultiScanResult scanResult = imsr.result;
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
            }
            ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
            for (TKeyValue kv : scanResult.results) {
                entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
            }
            if (!entries.isEmpty())
                receiver.receive(entries);
            if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
                timeoutTracker.madeProgress();
            trackScanning(failures, unscanned, scanResult);
            AtomicLong nextOpid = new AtomicLong();
            while (scanResult.more) {
                timeoutTracker.check();
                if (timer != null) {
                    log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
                    timer.reset().start();
                }
                scanResult = client.continueMultiScan(TraceUtil.traceInfo(), imsr.scanID);
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(SECONDS)));
                }
                entries = new ArrayList<>(scanResult.results.size());
                for (TKeyValue kv : scanResult.results) {
                    entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
                }
                if (!entries.isEmpty())
                    receiver.receive(entries);
                if (!entries.isEmpty() || !scanResult.fullScans.isEmpty())
                    timeoutTracker.madeProgress();
                trackScanning(failures, unscanned, scanResult);
            }
            client.closeMultiScan(TraceUtil.traceInfo(), imsr.scanID);
        } finally {
            ThriftUtil.returnClient(client, context);
        }
    } catch (TTransportException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage());
        timeoutTracker.errorOccured();
        throw new IOException(e);
    } catch (ThriftSecurityException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TApplicationException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloServerException(server, e);
    } catch (NoSuchScanIDException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new IOException(e);
    } catch (TSampleNotPresentException e) {
        log.debug("Server : " + server + " msg : " + e.getMessage(), e);
        String tableInfo = "?";
        if (e.getExtent() != null) {
            TableId tableId = KeyExtent.fromThrift(e.getExtent()).tableId();
            tableInfo = context.getPrintableTableInfoFromId(tableId);
        }
        String message = "Table " + tableInfo + " does not have sampling configured or built";
        throw new SampleNotPresentException(message, e);
    } catch (TException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        timeoutTracker.errorOccured();
        throw new IOException(e);
    }
}
Also used : TableId(org.apache.accumulo.core.data.TableId) ThriftUtil(org.apache.accumulo.core.rpc.ThriftUtil) ListIterator(java.util.ListIterator) TTransportException(org.apache.thrift.transport.TTransportException) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) LoggerFactory(org.slf4j.LoggerFactory) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Map(java.util.Map) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Value(org.apache.accumulo.core.data.Value) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) Column(org.apache.accumulo.core.data.Column) Collection(java.util.Collection) Set(java.util.Set) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) Collectors(java.util.stream.Collectors) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) List(java.util.List) Entry(java.util.Map.Entry) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) TraceUtil(org.apache.accumulo.core.trace.TraceUtil) ByteBufferUtil(org.apache.accumulo.core.util.ByteBufferUtil) TimedOutException(org.apache.accumulo.core.client.TimedOutException) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) HostAndPort(org.apache.accumulo.core.util.HostAndPort) HashMap(java.util.HashMap) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) TableDeletedException(org.apache.accumulo.core.client.TableDeletedException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) Key(org.apache.accumulo.core.data.Key) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) NoSuchElementException(java.util.NoSuchElementException) TApplicationException(org.apache.thrift.TApplicationException) ExecutorService(java.util.concurrent.ExecutorService) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Semaphore(java.util.concurrent.Semaphore) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) TException(org.apache.thrift.TException) IOException(java.io.IOException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) Authorizations(org.apache.accumulo.core.security.Authorizations) AccumuloException(org.apache.accumulo.core.client.AccumuloException) Range(org.apache.accumulo.core.data.Range) AtomicLong(java.util.concurrent.atomic.AtomicLong) OpTimer(org.apache.accumulo.core.util.OpTimer) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableId(org.apache.accumulo.core.data.TableId) TException(org.apache.thrift.TException) ArrayList(java.util.ArrayList) TTransportException(org.apache.thrift.transport.TTransportException) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) HostAndPort(org.apache.accumulo.core.util.HostAndPort) Entry(java.util.Map.Entry) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) Column(org.apache.accumulo.core.data.Column) List(java.util.List) ArrayList(java.util.ArrayList) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) IOException(java.io.IOException) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) Range(org.apache.accumulo.core.data.Range) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) TApplicationException(org.apache.thrift.TApplicationException) AtomicLong(java.util.concurrent.atomic.AtomicLong) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) OpTimer(org.apache.accumulo.core.util.OpTimer) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) Value(org.apache.accumulo.core.data.Value) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Key(org.apache.accumulo.core.data.Key)

Example 2 with MultiScanResult

use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.

the class TabletServerBatchReaderIterator method trackScanning.

static void trackScanning(Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, MultiScanResult scanResult) {
    // translate returned failures, remove them from unscanned, and add them to failures
    // @formatter:off
    Map<KeyExtent, List<Range>> retFailures = scanResult.failures.entrySet().stream().collect(Collectors.toMap((entry) -> KeyExtent.fromThrift(entry.getKey()), (entry) -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
    // @formatter:on
    unscanned.keySet().removeAll(retFailures.keySet());
    failures.putAll(retFailures);
    // translate full scans and remove them from unscanned
    Set<KeyExtent> fullScans = scanResult.fullScans.stream().map(KeyExtent::fromThrift).collect(Collectors.toSet());
    unscanned.keySet().removeAll(fullScans);
    // remove partial scan from unscanned
    if (scanResult.partScan != null) {
        KeyExtent ke = KeyExtent.fromThrift(scanResult.partScan);
        Key nextKey = new Key(scanResult.partNextKey);
        ListIterator<Range> iterator = unscanned.get(ke).listIterator();
        while (iterator.hasNext()) {
            Range range = iterator.next();
            if (range.afterEndKey(nextKey) || (nextKey.equals(range.getEndKey()) && scanResult.partNextKeyInclusive != range.isEndKeyInclusive())) {
                iterator.remove();
            } else if (range.contains(nextKey)) {
                iterator.remove();
                Range partRange = new Range(nextKey, scanResult.partNextKeyInclusive, range.getEndKey(), range.isEndKeyInclusive());
                iterator.add(partRange);
            }
        }
    }
}
Also used : TableId(org.apache.accumulo.core.data.TableId) ThriftUtil(org.apache.accumulo.core.rpc.ThriftUtil) ListIterator(java.util.ListIterator) TTransportException(org.apache.thrift.transport.TTransportException) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) LoggerFactory(org.slf4j.LoggerFactory) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Map(java.util.Map) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Value(org.apache.accumulo.core.data.Value) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) Column(org.apache.accumulo.core.data.Column) Collection(java.util.Collection) Set(java.util.Set) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) Collectors(java.util.stream.Collectors) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) List(java.util.List) Entry(java.util.Map.Entry) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) TraceUtil(org.apache.accumulo.core.trace.TraceUtil) ByteBufferUtil(org.apache.accumulo.core.util.ByteBufferUtil) TimedOutException(org.apache.accumulo.core.client.TimedOutException) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) HostAndPort(org.apache.accumulo.core.util.HostAndPort) HashMap(java.util.HashMap) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) TableDeletedException(org.apache.accumulo.core.client.TableDeletedException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) Key(org.apache.accumulo.core.data.Key) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) NoSuchElementException(java.util.NoSuchElementException) TApplicationException(org.apache.thrift.TApplicationException) ExecutorService(java.util.concurrent.ExecutorService) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Semaphore(java.util.concurrent.Semaphore) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) TException(org.apache.thrift.TException) IOException(java.io.IOException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) Authorizations(org.apache.accumulo.core.security.Authorizations) AccumuloException(org.apache.accumulo.core.client.AccumuloException) Range(org.apache.accumulo.core.data.Range) AtomicLong(java.util.concurrent.atomic.AtomicLong) OpTimer(org.apache.accumulo.core.util.OpTimer) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) List(java.util.List) ArrayList(java.util.ArrayList) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) Range(org.apache.accumulo.core.data.Range) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) Key(org.apache.accumulo.core.data.Key)

Example 3 with MultiScanResult

use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.

the class VerifyTabletAssignments method checkTabletServer.

private static void checkTabletServer(ClientContext context, Entry<HostAndPort, List<KeyExtent>> entry, HashSet<KeyExtent> failures) throws ThriftSecurityException, TException, NoSuchScanIDException {
    TabletClientService.Iface client = ThriftUtil.getTServerClient(entry.getKey(), context);
    Map<TKeyExtent, List<TRange>> batch = new TreeMap<>();
    for (KeyExtent keyExtent : entry.getValue()) {
        Text row = keyExtent.endRow();
        Text row2 = null;
        if (row == null) {
            row = keyExtent.prevEndRow();
            if (row != null) {
                row = new Text(row);
                row.append(new byte[] { 'a' }, 0, 1);
            } else {
                row = new Text("1234567890");
            }
            row2 = new Text(row);
            row2.append(new byte[] { '!' }, 0, 1);
        } else {
            row = new Text(row);
            row2 = new Text(row);
            row.getBytes()[row.getLength() - 1] = (byte) (row.getBytes()[row.getLength() - 1] - 1);
        }
        Range r = new Range(row, true, row2, false);
        batch.put(keyExtent.toThrift(), Collections.singletonList(r.toThrift()));
    }
    TInfo tinfo = TraceUtil.traceInfo();
    Map<String, Map<String, String>> emptyMapSMapSS = Collections.emptyMap();
    List<IterInfo> emptyListIterInfo = Collections.emptyList();
    List<TColumn> emptyListColumn = Collections.emptyList();
    InitialMultiScan is = client.startMultiScan(tinfo, context.rpcCreds(), batch, emptyListColumn, emptyListIterInfo, emptyMapSMapSS, Authorizations.EMPTY.getAuthorizationsBB(), false, null, 0L, null, null);
    if (is.result.more) {
        MultiScanResult result = client.continueMultiScan(tinfo, is.scanID);
        checkFailures(entry.getKey(), failures, result);
        while (result.more) {
            result = client.continueMultiScan(tinfo, is.scanID);
            checkFailures(entry.getKey(), failures, result);
        }
    }
    client.closeMultiScan(tinfo, is.scanID);
    ThriftUtil.returnClient((TServiceClient) client, context);
}
Also used : TInfo(org.apache.accumulo.core.trace.thrift.TInfo) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) Text(org.apache.hadoop.io.Text) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) TreeMap(java.util.TreeMap) Range(org.apache.accumulo.core.data.Range) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) IterInfo(org.apache.accumulo.core.dataImpl.thrift.IterInfo) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) ArrayList(java.util.ArrayList) List(java.util.List) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Map(java.util.Map) TreeMap(java.util.TreeMap)

Example 4 with MultiScanResult

use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.

the class ThriftClientHandler method startMultiScan.

@Override
public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent, List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws ThriftSecurityException, TSampleNotPresentException {
    // find all of the tables that need to be scanned
    final HashSet<TableId> tables = new HashSet<>();
    for (TKeyExtent keyExtent : tbatch.keySet()) {
        tables.add(TableId.of(new String(keyExtent.getTable(), UTF_8)));
    }
    if (tables.size() != 1) {
        throw new IllegalArgumentException("Cannot batch scan over multiple tables");
    }
    // check if user has permission to the tables
    for (TableId tableId : tables) {
        NamespaceId namespaceId = getNamespaceId(credentials, tableId);
        if (!security.canScan(credentials, tableId, namespaceId, tbatch, tcolumns, ssiList, ssio, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
        }
    }
    try {
        if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
        }
    } catch (ThriftSecurityException tse) {
        log.error("{} is not authorized", credentials.getPrincipal(), tse);
        throw tse;
    }
    // @formatter:off
    Map<KeyExtent, List<Range>> batch = tbatch.entrySet().stream().collect(Collectors.toMap(entry -> KeyExtent.fromThrift(entry.getKey()), entry -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
    // @formatter:on
    // This is used to determine which thread pool to use
    KeyExtent threadPoolExtent = batch.keySet().iterator().next();
    if (waitForWrites) {
        writeTracker.waitForWrites(TabletType.type(batch.keySet()));
    }
    Set<Column> columnSet = tcolumns.isEmpty() ? Collections.emptySet() : new HashSet<>(Collections2.transform(tcolumns, Column::new));
    ScanParameters scanParams = new ScanParameters(-1, new Authorizations(authorizations), columnSet, ssiList, ssio, false, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
    final MultiScanSession mss = new MultiScanSession(credentials, threadPoolExtent, batch, scanParams, executionHints);
    mss.numTablets = batch.size();
    for (List<Range> ranges : batch.values()) {
        mss.numRanges += ranges.size();
    }
    long sid = server.sessionManager.createSession(mss, true);
    MultiScanResult result;
    try {
        result = continueMultiScan(sid, mss);
    } finally {
        server.sessionManager.unreserveSession(sid);
    }
    return new InitialMultiScan(sid, result);
}
Also used : TableId(org.apache.accumulo.core.data.TableId) FileSystem(org.apache.hadoop.fs.FileSystem) TooManyFilesException(org.apache.accumulo.server.fs.TooManyFilesException) Text(org.apache.hadoop.io.Text) Collections2(com.google.common.collect.Collections2) ConstraintViolationSummary(org.apache.accumulo.core.data.ConstraintViolationSummary) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Future(java.util.concurrent.Future) TSummaries(org.apache.accumulo.core.dataImpl.thrift.TSummaries) CompressedIterators(org.apache.accumulo.core.clientImpl.CompressedIterators) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) ScanResult(org.apache.accumulo.core.dataImpl.thrift.ScanResult) Map(java.util.Map) RootTable(org.apache.accumulo.core.metadata.RootTable) MapFileInfo(org.apache.accumulo.core.dataImpl.thrift.MapFileInfo) BulkImportState(org.apache.accumulo.core.master.thrift.BulkImportState) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) Property(org.apache.accumulo.core.conf.Property) VolumeManager(org.apache.accumulo.server.fs.VolumeManager) CancellationException(java.util.concurrent.CancellationException) Column(org.apache.accumulo.core.data.Column) TSummaryRequest(org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest) TabletStats(org.apache.accumulo.core.tabletserver.thrift.TabletStats) PreparedMutations(org.apache.accumulo.tserver.tablet.PreparedMutations) TConditionalSession(org.apache.accumulo.core.dataImpl.thrift.TConditionalSession) Set(java.util.Set) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TCMStatus(org.apache.accumulo.core.dataImpl.thrift.TCMStatus) TCredentials(org.apache.accumulo.core.securityImpl.thrift.TCredentials) TExternalCompactionJob(org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob) TSamplerConfiguration(org.apache.accumulo.core.tabletserver.thrift.TSamplerConfiguration) ScanDispatcher(org.apache.accumulo.core.spi.scan.ScanDispatcher) RowLock(org.apache.accumulo.tserver.RowLocks.RowLock) ServiceLock(org.apache.accumulo.fate.zookeeper.ServiceLock) UtilWaitThread.sleepUninterruptibly(org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) KVEntry(org.apache.accumulo.tserver.tablet.KVEntry) SummaryCollection(org.apache.accumulo.core.summary.SummaryCollection) MINUTES(java.util.concurrent.TimeUnit.MINUTES) Threads(org.apache.accumulo.core.util.threads.Threads) ZooUtil(org.apache.accumulo.fate.zookeeper.ZooUtil) DurabilityImpl(org.apache.accumulo.core.clientImpl.DurabilityImpl) ArrayList(java.util.ArrayList) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) Gatherer(org.apache.accumulo.core.summary.Gatherer) ActiveCompaction(org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction) LookupTask(org.apache.accumulo.tserver.scan.LookupTask) FSError(org.apache.hadoop.fs.FSError) Key(org.apache.accumulo.core.data.Key) FileSystemResolver(org.apache.accumulo.core.summary.Gatherer.FileSystemResolver) CommitSession(org.apache.accumulo.tserver.tablet.CommitSession) TabletFile(org.apache.accumulo.core.metadata.TabletFile) InitialScan(org.apache.accumulo.core.dataImpl.thrift.InitialScan) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) ServerMutation(org.apache.accumulo.server.data.ServerMutation) TConditionalMutation(org.apache.accumulo.core.dataImpl.thrift.TConditionalMutation) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) ConstraintViolationException(org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException) TException(org.apache.thrift.TException) IOException(java.io.IOException) Range(org.apache.accumulo.core.data.Range) ExecutionException(java.util.concurrent.ExecutionException) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) ActiveScan(org.apache.accumulo.core.tabletserver.thrift.ActiveScan) TableId(org.apache.accumulo.core.data.TableId) FileCompactor(org.apache.accumulo.server.compaction.FileCompactor) NextBatchTask(org.apache.accumulo.tserver.scan.NextBatchTask) TabletServerStatus(org.apache.accumulo.core.master.thrift.TabletServerStatus) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MetadataTable(org.apache.accumulo.core.metadata.MetadataTable) TabletLogger(org.apache.accumulo.core.logging.TabletLogger) TServerUtils(org.apache.accumulo.server.rpc.TServerUtils) Mutation(org.apache.accumulo.core.data.Mutation) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) IterInfo(org.apache.accumulo.core.dataImpl.thrift.IterInfo) ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) ByteBuffer(java.nio.ByteBuffer) BlockCache(org.apache.accumulo.core.spi.cache.BlockCache) ConditionalSession(org.apache.accumulo.tserver.session.ConditionalSession) TInfo(org.apache.accumulo.core.trace.thrift.TInfo) UpdateSession(org.apache.accumulo.tserver.session.UpdateSession) TCMResult(org.apache.accumulo.core.dataImpl.thrift.TCMResult) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Path(org.apache.hadoop.fs.Path) TUnloadTabletGoal(org.apache.accumulo.core.tabletserver.thrift.TUnloadTabletGoal) SingleScanSession(org.apache.accumulo.tserver.session.SingleScanSession) ConditionChecker(org.apache.accumulo.tserver.ConditionCheckerContext.ConditionChecker) ServerConditionalMutation(org.apache.accumulo.tserver.data.ServerConditionalMutation) Span(io.opentelemetry.api.trace.Span) Collection(java.util.Collection) Durability(org.apache.accumulo.core.client.Durability) TransactionWatcher(org.apache.accumulo.server.zookeeper.TransactionWatcher) Collectors(java.util.stream.Collectors) TabletClosedException(org.apache.accumulo.tserver.tablet.TabletClosedException) List(java.util.List) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) Pair(org.apache.accumulo.core.util.Pair) Entry(java.util.Map.Entry) NamespaceId(org.apache.accumulo.core.data.NamespaceId) TraceUtil(org.apache.accumulo.core.trace.TraceUtil) ByteBufferUtil(org.apache.accumulo.core.util.ByteBufferUtil) IterationInterruptedException(org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException) TRowRange(org.apache.accumulo.core.dataImpl.thrift.TRowRange) TCompactionQueueSummary(org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary) HashMap(java.util.HashMap) CompactionConfig(org.apache.accumulo.core.client.admin.CompactionConfig) HashSet(java.util.HashSet) NotServingTabletException(org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException) SummarySession(org.apache.accumulo.tserver.session.SummarySession) Halt(org.apache.accumulo.core.util.Halt) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) TableOperationExceptionType(org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType) ExecutorService(java.util.concurrent.ExecutorService) UpdateErrors(org.apache.accumulo.core.dataImpl.thrift.UpdateErrors) TMutation(org.apache.accumulo.core.dataImpl.thrift.TMutation) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) SecurityErrorCode(org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode) Scope(io.opentelemetry.context.Scope) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ClientServiceHandler(org.apache.accumulo.server.client.ClientServiceHandler) CompactionInfo(org.apache.accumulo.server.compaction.CompactionInfo) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) Constants(org.apache.accumulo.core.Constants) Authorizations(org.apache.accumulo.core.security.Authorizations) ScanBatch(org.apache.accumulo.tserver.tablet.ScanBatch) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) ThriftTableOperationException(org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException) TabletType(org.apache.accumulo.core.clientImpl.TabletType) TDurability(org.apache.accumulo.core.tabletserver.thrift.TDurability) Tablet(org.apache.accumulo.tserver.tablet.Tablet) Cache(com.google.common.cache.Cache) Collections(java.util.Collections) Authorizations(org.apache.accumulo.core.security.Authorizations) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) Range(org.apache.accumulo.core.data.Range) TRowRange(org.apache.accumulo.core.dataImpl.thrift.TRowRange) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) Column(org.apache.accumulo.core.data.Column) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) NamespaceId(org.apache.accumulo.core.data.NamespaceId) HashSet(java.util.HashSet)

Example 5 with MultiScanResult

use of org.apache.accumulo.core.dataImpl.thrift.MultiScanResult in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.isEmpty()) {
                fullScans.add(entry.getKey());
            } else {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        // @formatter:off
        Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
        // @formatter:on
        List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Exception e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}
Also used : IterationInterruptedException(org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) KVEntry(org.apache.accumulo.tserver.tablet.KVEntry) LookupResult(org.apache.accumulo.tserver.tablet.Tablet.LookupResult) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) Map(java.util.Map) Key(org.apache.accumulo.core.data.Key) Property(org.apache.accumulo.core.conf.Property) TabletServer(org.apache.accumulo.tserver.TabletServer) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) IOException(java.io.IOException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) Collectors(java.util.stream.Collectors) Range(org.apache.accumulo.core.data.Range) List(java.util.List) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) Entry(java.util.Map.Entry) Tablet(org.apache.accumulo.tserver.tablet.Tablet) TKey(org.apache.accumulo.core.dataImpl.thrift.TKey) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TKey(org.apache.accumulo.core.dataImpl.thrift.TKey) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) KVEntry(org.apache.accumulo.tserver.tablet.KVEntry) Entry(java.util.Map.Entry) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) IterationInterruptedException(org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException) ArrayList(java.util.ArrayList) List(java.util.List) Tablet(org.apache.accumulo.tserver.tablet.Tablet) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) IOException(java.io.IOException) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) Range(org.apache.accumulo.core.data.Range) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) IterationInterruptedException(org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException) IOException(java.io.IOException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) KVEntry(org.apache.accumulo.tserver.tablet.KVEntry) LookupResult(org.apache.accumulo.tserver.tablet.Tablet.LookupResult) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) Key(org.apache.accumulo.core.data.Key) TKey(org.apache.accumulo.core.dataImpl.thrift.TKey)

Aggregations

ArrayList (java.util.ArrayList)6 List (java.util.List)6 Map (java.util.Map)6 MultiScanResult (org.apache.accumulo.core.dataImpl.thrift.MultiScanResult)6 TRange (org.apache.accumulo.core.dataImpl.thrift.TRange)6 IOException (java.io.IOException)5 HashMap (java.util.HashMap)5 SampleNotPresentException (org.apache.accumulo.core.client.SampleNotPresentException)5 Range (org.apache.accumulo.core.data.Range)5 KeyExtent (org.apache.accumulo.core.dataImpl.KeyExtent)5 TKeyExtent (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)5 Iterator (java.util.Iterator)4 Entry (java.util.Map.Entry)4 Collectors (java.util.stream.Collectors)4 AccumuloSecurityException (org.apache.accumulo.core.client.AccumuloSecurityException)4 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)4 ThriftSecurityException (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)4 Key (org.apache.accumulo.core.data.Key)4 InitialMultiScan (org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan)4 TKeyValue (org.apache.accumulo.core.dataImpl.thrift.TKeyValue)4