Examples with KeyValue org.apache.accumulo.core.data.KeyValue used on opensource projects

Example 6 with KeyValue

use of org.apache.accumulo.core.data.KeyValue in project accumulo-examples by apache.

the class ChunkInputStreamIT method addData.

static void addData(List<Entry<Key, Value>> data, String row, String cf, int chunkSize, int chunkCount, String vis, String value) {
    Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
    chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
    data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)), value.getBytes()));
}

Example 7 with KeyValue

use of org.apache.accumulo.core.data.KeyValue in project accumulo by apache.

the class ThriftScanner method scan.

public static List<KeyValue> scan(ClientContext context, ScanState scanState, int timeOut) throws ScanTimedOutException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
    TabletLocation loc = null;
    Instance instance = context.getInstance();
    long startTime = System.currentTimeMillis();
    String lastError = null;
    String error = null;
    int tooManyFilesCount = 0;
    long sleepMillis = 100;
    final long maxSleepTime = context.getConfiguration().getTimeInMillis(Property.GENERAL_MAX_SCANNER_RETRY_PERIOD);
    List<KeyValue> results = null;
    Span span = Trace.start("scan");
    try {
        while (results == null && !scanState.finished) {
            if (Thread.currentThread().isInterrupted()) {
                throw new AccumuloException("Thread interrupted");
            }
            if ((System.currentTimeMillis() - startTime) / 1000.0 > timeOut)
                throw new ScanTimedOutException();
            while (loc == null) {
                long currentTime = System.currentTimeMillis();
                if ((currentTime - startTime) / 1000.0 > timeOut)
                    throw new ScanTimedOutException();
                Span locateSpan = Trace.start("scan:locateTablet");
                try {
                    loc = TabletLocator.getLocator(context, scanState.tableId).locateTablet(context, scanState.startRow, scanState.skipStartRow, false);
                    if (loc == null) {
                        if (!Tables.exists(instance, scanState.tableId))
                            throw new TableDeletedException(scanState.tableId.canonicalID());
                        else if (Tables.getTableState(instance, scanState.tableId) == TableState.OFFLINE)
                            throw new TableOfflineException(instance, scanState.tableId.canonicalID());
                        error = "Failed to locate tablet for table : " + scanState.tableId + " row : " + scanState.startRow;
                        if (!error.equals(lastError))
                            log.debug("{}", error);
                        else if (log.isTraceEnabled())
                            log.trace("{}", error);
                        lastError = error;
                        sleepMillis = pause(sleepMillis, maxSleepTime);
                    } else {
                        // when a tablet splits we do want to continue scanning the low child
                        // of the split if we are already passed it
                        Range dataRange = loc.tablet_extent.toDataRange();
                        if (scanState.range.getStartKey() != null && dataRange.afterEndKey(scanState.range.getStartKey())) {
                            // go to the next tablet
                            scanState.startRow = loc.tablet_extent.getEndRow();
                            scanState.skipStartRow = true;
                            loc = null;
                        } else if (scanState.range.getEndKey() != null && dataRange.beforeStartKey(scanState.range.getEndKey())) {
                            // should not happen
                            throw new RuntimeException("Unexpected tablet, extent : " + loc.tablet_extent + "  range : " + scanState.range + " startRow : " + scanState.startRow);
                        }
                    }
                } catch (AccumuloServerException e) {
                    log.debug("Scan failed, server side exception : {}", e.getMessage());
                    throw e;
                } catch (AccumuloException e) {
                    error = "exception from tablet loc " + e.getMessage();
                    if (!error.equals(lastError))
                        log.debug("{}", error);
                    else if (log.isTraceEnabled())
                        log.trace("{}", error);
                    lastError = error;
                    sleepMillis = pause(sleepMillis, maxSleepTime);
                } finally {
                    locateSpan.stop();
                }
            }
            Span scanLocation = Trace.start("scan:location");
            scanLocation.data("tserver", loc.tablet_location);
            try {
                results = scan(loc, scanState, context);
            } catch (AccumuloSecurityException e) {
                Tables.clearCache(instance);
                if (!Tables.exists(instance, scanState.tableId))
                    throw new TableDeletedException(scanState.tableId.canonicalID());
                e.setTableInfo(Tables.getPrintableTableInfoFromId(instance, scanState.tableId));
                throw e;
            } catch (TApplicationException tae) {
                throw new AccumuloServerException(loc.tablet_location, tae);
            } catch (TSampleNotPresentException tsnpe) {
                String message = "Table " + Tables.getPrintableTableInfoFromId(instance, scanState.tableId) + " does not have sampling configured or built";
                throw new SampleNotPresentException(message, tsnpe);
            } catch (NotServingTabletException e) {
                error = "Scan failed, not serving tablet " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                TabletLocator.getLocator(context, scanState.tableId).invalidateCache(loc.tablet_extent);
                loc = null;
                // no need to try the current scan id somewhere else
                scanState.scanID = null;
                if (scanState.isolated)
                    throw new IsolationException();
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } catch (NoSuchScanIDException e) {
                error = "Scan failed, no such scan id " + scanState.scanID + " " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                if (scanState.isolated)
                    throw new IsolationException();
                scanState.scanID = null;
            } catch (TooManyFilesException e) {
                error = "Tablet has too many files " + loc + " retrying...";
                if (!error.equals(lastError)) {
                    log.debug("{}", error);
                    tooManyFilesCount = 0;
                } else {
                    tooManyFilesCount++;
                    if (tooManyFilesCount == 300)
                        log.warn("{}", error);
                    else if (log.isTraceEnabled())
                        log.trace("{}", error);
                }
                lastError = error;
                // not sure what state the scan session on the server side is
                // in after this occurs, so lets be cautious and start a new
                // scan session
                scanState.scanID = null;
                if (scanState.isolated)
                    throw new IsolationException();
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } catch (TException e) {
                TabletLocator.getLocator(context, scanState.tableId).invalidateCache(context.getInstance(), loc.tablet_location);
                error = "Scan failed, thrift error " + e.getClass().getName() + "  " + e.getMessage() + " " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                loc = null;
                // do not want to continue using the same scan id, if a timeout occurred could cause a batch to be skipped
                // because a thread on the server side may still be processing the timed out continue scan
                scanState.scanID = null;
                if (scanState.isolated)
                    throw new IsolationException();
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } finally {
                scanLocation.stop();
            }
        }
        if (results != null && results.size() == 0 && scanState.finished) {
            results = null;
        }
        return results;
    } catch (InterruptedException ex) {
        throw new AccumuloException(ex);
    } finally {
        span.stop();
    }
}

Example 8 with KeyValue

use of org.apache.accumulo.core.data.KeyValue in project accumulo by apache.

the class ThriftScanner method scan.

private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, ClientContext context) throws AccumuloSecurityException, NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException, TSampleNotPresentException {
    if (scanState.finished)
        return null;
    OpTimer timer = null;
    final TInfo tinfo = Tracer.traceInfo();
    final HostAndPort parsedLocation = HostAndPort.fromString(loc.tablet_location);
    TabletClientService.Client client = ThriftUtil.getTServerClient(parsedLocation, context);
    String old = Thread.currentThread().getName();
    try {
        ScanResult sr;
        if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc))
            scanState.scanID = null;
        scanState.prevLoc = loc;
        if (scanState.scanID == null) {
            Thread.currentThread().setName("Starting scan tserver=" + loc.tablet_location + " tableId=" + loc.tablet_extent.getTableId());
            if (log.isTraceEnabled()) {
                String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil=" + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions + " context=" + scanState.classLoaderContext;
                log.trace("tid={} {}", Thread.currentThread().getId(), msg);
                timer = new OpTimer().start();
            }
            TabletType ttype = TabletType.type(loc.tablet_extent);
            boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
            InitialScan is = client.startScan(tinfo, scanState.context.rpcCreds(), loc.tablet_extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, SamplerConfigurationImpl.toThrift(scanState.samplerConfig), scanState.batchTimeOut, scanState.classLoaderContext);
            if (waitForWrites)
                serversWaitedForWrites.get(ttype).add(loc.tablet_location);
            sr = is.result;
            if (sr.more)
                scanState.scanID = is.scanID;
            else
                client.closeScan(tinfo, is.scanID);
        } else {
            // log.debug("Calling continue scan : "+scanState.range+" loc = "+loc);
            String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
            Thread.currentThread().setName(msg);
            if (log.isTraceEnabled()) {
                log.trace("tid={} {}", Thread.currentThread().getId(), msg);
                timer = new OpTimer().start();
            }
            sr = client.continueScan(tinfo, scanState.scanID);
            if (!sr.more) {
                client.closeScan(tinfo, scanState.scanID);
                scanState.scanID = null;
            }
        }
        if (!sr.more) {
            // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = "+scanState.range);
            if (loc.tablet_extent.getEndRow() == null) {
                scanState.finished = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Completely finished scan in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey(new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
                scanState.startRow = loc.tablet_extent.getEndRow();
                scanState.skipStartRow = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Finished scanning tablet in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            } else {
                scanState.finished = true;
                if (timer != null) {
                    timer.stop();
                    log.trace("tid={} Completely finished in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size());
                }
            }
        } else {
            if (timer != null) {
                timer.stop();
                log.trace("tid={} Finished scan in {} #results={} scanid={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size(), scanState.scanID);
            }
        }
        Key.decompress(sr.results);
        if (sr.results.size() > 0 && !scanState.finished)
            scanState.range = new Range(new Key(sr.results.get(sr.results.size() - 1).key), false, scanState.range.getEndKey(), scanState.range.isEndKeyInclusive());
        List<KeyValue> results = new ArrayList<>(sr.results.size());
        for (TKeyValue tkv : sr.results) results.add(new KeyValue(new Key(tkv.key), tkv.value));
        return results;
    } catch (ThriftSecurityException e) {
        throw new AccumuloSecurityException(e.user, e.code, e);
    } finally {
        ThriftUtil.returnClient(client);
        Thread.currentThread().setName(old);
    }
}