Examples with KeyExtent org.apache.accumulo.core.data.impl.KeyExtent used on opensource projects

Example 81 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TableOperationsImpl method waitForTableStateTransition.

private void waitForTableStateTransition(Table.ID tableId, TableState expectedState) throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    Text startRow = null;
    Text lastRow = null;
    while (true) {
        if (Tables.getTableState(context.getInstance(), tableId) != expectedState) {
            Tables.clearCache(context.getInstance());
            TableState currentState = Tables.getTableState(context.getInstance(), tableId);
            if (currentState != expectedState) {
                if (!Tables.exists(context.getInstance(), tableId))
                    throw new TableDeletedException(tableId.canonicalID());
                if (currentState == TableState.DELETING)
                    throw new TableNotFoundException(tableId.canonicalID(), "", "Table is being deleted.");
                throw new AccumuloException("Unexpected table state " + tableId + " " + Tables.getTableState(context.getInstance(), tableId) + " != " + expectedState);
            }
        }
        Range range;
        if (startRow == null || lastRow == null)
            range = new KeyExtent(tableId, null, null).toMetadataRange();
        else
            range = new Range(startRow, lastRow);
        String metaTable = MetadataTable.NAME;
        if (tableId.equals(MetadataTable.ID))
            metaTable = RootTable.NAME;
        Scanner scanner = createMetadataScanner(metaTable, range);
        RowIterator rowIter = new RowIterator(scanner);
        KeyExtent lastExtent = null;
        int total = 0;
        int waitFor = 0;
        int holes = 0;
        Text continueRow = null;
        MapCounter<String> serverCounts = new MapCounter<>();
        while (rowIter.hasNext()) {
            Iterator<Entry<Key, Value>> row = rowIter.next();
            total++;
            KeyExtent extent = null;
            String future = null;
            String current = null;
            while (row.hasNext()) {
                Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME))
                    future = entry.getValue().toString();
                if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME))
                    current = entry.getValue().toString();
                if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key))
                    extent = new KeyExtent(key.getRow(), entry.getValue());
            }
            if ((expectedState == TableState.ONLINE && current == null) || (expectedState == TableState.OFFLINE && (future != null || current != null))) {
                if (continueRow == null)
                    continueRow = extent.getMetadataEntry();
                waitFor++;
                lastRow = extent.getMetadataEntry();
                if (current != null)
                    serverCounts.increment(current, 1);
                if (future != null)
                    serverCounts.increment(future, 1);
            }
            if (!extent.getTableId().equals(tableId)) {
                throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
            }
            if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
                holes++;
            }
            lastExtent = extent;
        }
        if (continueRow != null) {
            startRow = continueRow;
        }
        if (holes > 0 || total == 0) {
            startRow = null;
            lastRow = null;
        }
        if (waitFor > 0 || holes > 0 || total == 0) {
            long waitTime;
            long maxPerServer = 0;
            if (serverCounts.size() > 0) {
                maxPerServer = Collections.max(serverCounts.values());
                waitTime = maxPerServer * 10;
            } else
                waitTime = waitFor * 10;
            waitTime = Math.max(100, waitTime);
            waitTime = Math.min(5000, waitTime);
            log.trace("Waiting for {}({}) tablets, startRow = {} lastRow = {}, holes={} sleeping:{}ms", waitFor, maxPerServer, startRow, lastRow, holes, waitTime);
            sleepUninterruptibly(waitTime, TimeUnit.MILLISECONDS);
        } else {
            break;
        }
    }
}

Example 82 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletLocatorImpl method processInvalidated.

private void processInvalidated(ClientContext context, LockCheckerSession lcSession) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
    if (badExtents.size() == 0)
        return;
    final boolean writeLockHeld = rwLock.isWriteLockedByCurrentThread();
    try {
        if (!writeLockHeld) {
            rLock.unlock();
            wLock.lock();
            if (badExtents.size() == 0)
                return;
        }
        List<Range> lookups = new ArrayList<>(badExtents.size());
        for (KeyExtent be : badExtents) {
            lookups.add(be.toMetadataRange());
            removeOverlapping(metaCache, be);
        }
        lookups = Range.mergeOverlapping(lookups);
        Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
        parent.binRanges(context, lookups, binnedRanges);
        // randomize server order
        ArrayList<String> tabletServers = new ArrayList<>(binnedRanges.keySet());
        Collections.shuffle(tabletServers);
        for (String tserver : tabletServers) {
            List<TabletLocation> locations = locationObtainer.lookupTablets(context, tserver, binnedRanges.get(tserver), parent);
            for (TabletLocation tabletLocation : locations) {
                updateCache(tabletLocation, lcSession);
            }
        }
    } finally {
        if (!writeLockHeld) {
            rLock.lock();
            wLock.unlock();
        }
    }
}

Example 83 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletLocatorImpl method lookupTabletLocation.

private void lookupTabletLocation(ClientContext context, Text row, boolean retry, LockCheckerSession lcSession) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    Text metadataRow = new Text(tableId.getUtf8());
    metadataRow.append(new byte[] { ';' }, 0, 1);
    metadataRow.append(row.getBytes(), 0, row.getLength());
    TabletLocation ptl = parent.locateTablet(context, metadataRow, false, retry);
    if (ptl != null) {
        TabletLocations locations = locationObtainer.lookupTablet(context, ptl, metadataRow, lastTabletRow, parent);
        while (locations != null && locations.getLocations().isEmpty() && locations.getLocationless().isEmpty()) {
            // try the next tablet, the current tablet does not have any tablets that overlap the row
            Text er = ptl.tablet_extent.getEndRow();
            if (er != null && er.compareTo(lastTabletRow) < 0) {
                // System.out.println("er "+er+"  ltr "+lastTabletRow);
                ptl = parent.locateTablet(context, er, true, retry);
                if (ptl != null)
                    locations = locationObtainer.lookupTablet(context, ptl, metadataRow, lastTabletRow, parent);
                else
                    break;
            } else {
                break;
            }
        }
        if (locations == null)
            return;
        // cannot assume the list contains contiguous key extents... so it is probably
        // best to deal with each extent individually
        Text lastEndRow = null;
        for (TabletLocation tabletLocation : locations.getLocations()) {
            KeyExtent ke = tabletLocation.tablet_extent;
            TabletLocation locToCache;
            // create new location if current prevEndRow == endRow
            if ((lastEndRow != null) && (ke.getPrevEndRow() != null) && ke.getPrevEndRow().equals(lastEndRow)) {
                locToCache = new TabletLocation(new KeyExtent(ke.getTableId(), ke.getEndRow(), lastEndRow), tabletLocation.tablet_location, tabletLocation.tablet_session);
            } else {
                locToCache = tabletLocation;
            }
            // save endRow for next iteration
            lastEndRow = locToCache.tablet_extent.getEndRow();
            updateCache(locToCache, lcSession);
        }
    }
}

Example 84 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletServerBatchReaderIterator method doLookup.

static void doLookup(ClientContext context, String server, Map<KeyExtent, List<Range>> requested, Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, ResultReceiver receiver, List<Column> columns, ScannerOptions options, Authorizations authorizations, TimeoutTracker timeoutTracker) throws IOException, AccumuloSecurityException, AccumuloServerException {
    if (requested.size() == 0) {
        return;
    }
    // copy requested to unscanned map. we will remove ranges as they are scanned in trackScanning()
    for (Entry<KeyExtent, List<Range>> entry : requested.entrySet()) {
        ArrayList<Range> ranges = new ArrayList<>();
        for (Range range : entry.getValue()) {
            ranges.add(new Range(range));
        }
        unscanned.put(new KeyExtent(entry.getKey()), ranges);
    }
    timeoutTracker.startingScan();
    TTransport transport = null;
    try {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        final TabletClientService.Client client;
        if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
            client = ThriftUtil.getTServerClient(parsedServer, context, timeoutTracker.getTimeOut());
        else
            client = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            OpTimer timer = null;
            if (log.isTraceEnabled()) {
                log.trace("tid={} Starting multi scan, tserver={}  #tablets={}  #ranges={} ssil={} ssio={}", Thread.currentThread().getId(), server, requested.size(), sumSizes(requested.values()), options.serverSideIteratorList, options.serverSideIteratorOptions);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(requested.keySet());
            boolean waitForWrites = !ThriftScanner.serversWaitedForWrites.get(ttype).contains(server);
            Map<TKeyExtent, List<TRange>> thriftTabletRanges = Translator.translate(requested, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
            InitialMultiScan imsr = client.startMultiScan(Tracer.traceInfo(), context.rpcCreds(), thriftTabletRanges, Translator.translate(columns, Translators.CT), options.serverSideIteratorList, options.serverSideIteratorOptions, ByteBufferUtil.toByteBuffers(authorizations.getAuthorizations()), waitForWrites, SamplerConfigurationImpl.toThrift(options.getSamplerConfiguration()), options.batchTimeOut, options.classLoaderContext);
            if (waitForWrites)
                ThriftScanner.serversWaitedForWrites.get(ttype).add(server.toString());
            MultiScanResult scanResult = imsr.result;
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Got 1st multi scan results, #results={} {} in {}", Thread.currentThread().getId(), scanResult.results.size(), (scanResult.more ? "scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
            }
            ArrayList<Entry<Key, Value>> entries = new ArrayList<>(scanResult.results.size());
            for (TKeyValue kv : scanResult.results) {
                entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
            }
            if (entries.size() > 0)
                receiver.receive(entries);
            if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                timeoutTracker.madeProgress();
            trackScanning(failures, unscanned, scanResult);
            AtomicLong nextOpid = new AtomicLong();
            while (scanResult.more) {
                timeoutTracker.check();
                if (timer != null) {
                    log.trace("tid={} oid={} Continuing multi scan, scanid={}", Thread.currentThread().getId(), nextOpid.get(), imsr.scanID);
                    timer.reset().start();
                }
                scanResult = client.continueMultiScan(Tracer.traceInfo(), imsr.scanID);
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} oid={} Got more multi scan results, #results={} {} in {}", Thread.currentThread().getId(), nextOpid.getAndIncrement(), scanResult.results.size(), (scanResult.more ? " scanID=" + imsr.scanID : ""), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)));
                }
                entries = new ArrayList<>(scanResult.results.size());
                for (TKeyValue kv : scanResult.results) {
                    entries.add(new SimpleImmutableEntry<>(new Key(kv.key), new Value(kv.value)));
                }
                if (entries.size() > 0)
                    receiver.receive(entries);
                if (entries.size() > 0 || scanResult.fullScans.size() > 0)
                    timeoutTracker.madeProgress();
                trackScanning(failures, unscanned, scanResult);
            }
            client.closeMultiScan(Tracer.traceInfo(), imsr.scanID);
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (TTransportException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage());
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } catch (ThriftSecurityException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TApplicationException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new AccumuloServerException(server, e);
    } catch (NoSuchScanIDException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        throw new IOException(e);
    } catch (TSampleNotPresentException e) {
        log.debug("Server : " + server + " msg : " + e.getMessage(), e);
        String tableInfo = "?";
        if (e.getExtent() != null) {
            Table.ID tableId = new KeyExtent(e.getExtent()).getTableId();
            tableInfo = Tables.getPrintableTableInfoFromId(context.getInstance(), tableId);
        }
        String message = "Table " + tableInfo + " does not have sampling configured or built";
        throw new SampleNotPresentException(message, e);
    } catch (TException e) {
        log.debug("Server : {} msg : {}", server, e.getMessage(), e);
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
    } finally {
        ThriftTransportPool.getInstance().returnTransport(transport);
    }
}

Example 85 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletServerBatchReaderIterator method trackScanning.

static void trackScanning(Map<KeyExtent, List<Range>> failures, Map<KeyExtent, List<Range>> unscanned, MultiScanResult scanResult) {
    // translate returned failures, remove them from unscanned, and add them to failures
    Map<KeyExtent, List<Range>> retFailures = Translator.translate(scanResult.failures, Translators.TKET, new Translator.ListTranslator<>(Translators.TRT));
    unscanned.keySet().removeAll(retFailures.keySet());
    failures.putAll(retFailures);
    // translate full scans and remove them from unscanned
    HashSet<KeyExtent> fullScans = new HashSet<>(Translator.translate(scanResult.fullScans, Translators.TKET));
    unscanned.keySet().removeAll(fullScans);
    // remove partial scan from unscanned
    if (scanResult.partScan != null) {
        KeyExtent ke = new KeyExtent(scanResult.partScan);
        Key nextKey = new Key(scanResult.partNextKey);
        ListIterator<Range> iterator = unscanned.get(ke).listIterator();
        while (iterator.hasNext()) {
            Range range = iterator.next();
            if (range.afterEndKey(nextKey) || (nextKey.equals(range.getEndKey()) && scanResult.partNextKeyInclusive != range.isEndKeyInclusive())) {
                iterator.remove();
            } else if (range.contains(nextKey)) {
                iterator.remove();
                Range partRange = new Range(nextKey, scanResult.partNextKeyInclusive, range.getEndKey(), range.isEndKeyInclusive());
                iterator.add(partRange);
            }
        }
    }
}