Examples with TabletLocation org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation used on opensource projects

Example 1 with TabletLocation

use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.

the class ThriftScanner method scan.

public static List<KeyValue> scan(ClientContext context, ScanState scanState, long timeOut) throws ScanTimedOutException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
    TabletLocation loc = null;
    long startTime = System.currentTimeMillis();
    String lastError = null;
    String error = null;
    int tooManyFilesCount = 0;
    long sleepMillis = 100;
    final long maxSleepTime = context.getConfiguration().getTimeInMillis(Property.GENERAL_MAX_SCANNER_RETRY_PERIOD);
    List<KeyValue> results = null;
    Span parent = TraceUtil.startSpan(ThriftScanner.class, "scan");
    try (Scope scope = parent.makeCurrent()) {
        while (results == null && !scanState.finished) {
            if (Thread.currentThread().isInterrupted()) {
                throw new AccumuloException("Thread interrupted");
            }
            if ((System.currentTimeMillis() - startTime) / 1000.0 > timeOut)
                throw new ScanTimedOutException();
            while (loc == null) {
                long currentTime = System.currentTimeMillis();
                if ((currentTime - startTime) / 1000.0 > timeOut)
                    throw new ScanTimedOutException();
                Span child1 = TraceUtil.startSpan(ThriftScanner.class, "scan::locateTablet");
                try (Scope locateSpan = child1.makeCurrent()) {
                    loc = TabletLocator.getLocator(context, scanState.tableId).locateTablet(context, scanState.startRow, scanState.skipStartRow, false);
                    if (loc == null) {
                        context.requireNotDeleted(scanState.tableId);
                        context.requireNotOffline(scanState.tableId, null);
                        error = "Failed to locate tablet for table : " + scanState.tableId + " row : " + scanState.startRow;
                        if (!error.equals(lastError))
                            log.debug("{}", error);
                        else if (log.isTraceEnabled())
                            log.trace("{}", error);
                        lastError = error;
                        sleepMillis = pause(sleepMillis, maxSleepTime);
                    } else {
                        // when a tablet splits we do want to continue scanning the low child
                        // of the split if we are already passed it
                        Range dataRange = loc.tablet_extent.toDataRange();
                        if (scanState.range.getStartKey() != null && dataRange.afterEndKey(scanState.range.getStartKey())) {
                            // go to the next tablet
                            scanState.startRow = loc.tablet_extent.endRow();
                            scanState.skipStartRow = true;
                            loc = null;
                        } else if (scanState.range.getEndKey() != null && dataRange.beforeStartKey(scanState.range.getEndKey())) {
                            // should not happen
                            throw new RuntimeException("Unexpected tablet, extent : " + loc.tablet_extent + "  range : " + scanState.range + " startRow : " + scanState.startRow);
                        }
                    }
                } catch (AccumuloServerException e) {
                    TraceUtil.setException(child1, e, true);
                    log.debug("Scan failed, server side exception : {}", e.getMessage());
                    throw e;
                } catch (AccumuloException e) {
                    error = "exception from tablet loc " + e.getMessage();
                    if (!error.equals(lastError))
                        log.debug("{}", error);
                    else if (log.isTraceEnabled())
                        log.trace("{}", error);
                    TraceUtil.setException(child1, e, false);
                    lastError = error;
                    sleepMillis = pause(sleepMillis, maxSleepTime);
                } finally {
                    child1.end();
                }
            }
            Span child2 = TraceUtil.startSpan(ThriftScanner.class, "scan::location", Map.of("tserver", loc.tablet_location));
            try (Scope scanLocation = child2.makeCurrent()) {
                results = scan(loc, scanState, context);
            } catch (AccumuloSecurityException e) {
                context.clearTableListCache();
                context.requireNotDeleted(scanState.tableId);
                e.setTableInfo(context.getPrintableTableInfoFromId(scanState.tableId));
                TraceUtil.setException(child2, e, true);
                throw e;
            } catch (TApplicationException tae) {
                TraceUtil.setException(child2, tae, true);
                throw new AccumuloServerException(loc.tablet_location, tae);
            } catch (TSampleNotPresentException tsnpe) {
                String message = "Table " + context.getPrintableTableInfoFromId(scanState.tableId) + " does not have sampling configured or built";
                TraceUtil.setException(child2, tsnpe, true);
                throw new SampleNotPresentException(message, tsnpe);
            } catch (NotServingTabletException e) {
                error = "Scan failed, not serving tablet " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                TabletLocator.getLocator(context, scanState.tableId).invalidateCache(loc.tablet_extent);
                loc = null;
                // no need to try the current scan id somewhere else
                scanState.scanID = null;
                if (scanState.isolated) {
                    TraceUtil.setException(child2, e, true);
                    throw new IsolationException();
                }
                TraceUtil.setException(child2, e, false);
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } catch (NoSuchScanIDException e) {
                error = "Scan failed, no such scan id " + scanState.scanID + " " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                if (scanState.isolated) {
                    TraceUtil.setException(child2, e, true);
                    throw new IsolationException();
                }
                TraceUtil.setException(child2, e, false);
                scanState.scanID = null;
            } catch (TooManyFilesException e) {
                error = "Tablet has too many files " + loc + " retrying...";
                if (error.equals(lastError)) {
                    tooManyFilesCount++;
                    if (tooManyFilesCount == 300)
                        log.warn("{}", error);
                    else if (log.isTraceEnabled())
                        log.trace("{}", error);
                } else {
                    log.debug("{}", error);
                    tooManyFilesCount = 0;
                }
                lastError = error;
                // not sure what state the scan session on the server side is
                // in after this occurs, so lets be cautious and start a new
                // scan session
                scanState.scanID = null;
                if (scanState.isolated) {
                    TraceUtil.setException(child2, e, true);
                    throw new IsolationException();
                }
                TraceUtil.setException(child2, e, false);
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } catch (TException e) {
                TabletLocator.getLocator(context, scanState.tableId).invalidateCache(context, loc.tablet_location);
                error = "Scan failed, thrift error " + e.getClass().getName() + "  " + e.getMessage() + " " + loc;
                if (!error.equals(lastError))
                    log.debug("{}", error);
                else if (log.isTraceEnabled())
                    log.trace("{}", error);
                lastError = error;
                loc = null;
                // do not want to continue using the same scan id, if a timeout occurred could cause a
                // batch to be skipped
                // because a thread on the server side may still be processing the timed out continue scan
                scanState.scanID = null;
                if (scanState.isolated) {
                    TraceUtil.setException(child2, e, true);
                    throw new IsolationException();
                }
                TraceUtil.setException(child2, e, false);
                sleepMillis = pause(sleepMillis, maxSleepTime);
            } finally {
                child2.end();
            }
        }
        if (results != null && results.isEmpty() && scanState.finished) {
            results = null;
        }
        return results;
    } catch (InterruptedException ex) {
        TraceUtil.setException(parent, ex, true);
        throw new AccumuloException(ex);
    } finally {
        parent.end();
    }
}

Example 2 with TabletLocation

use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.

the class MetadataLocationObtainer method lookupTablets.

@Override
public List<TabletLocation> lookupTablets(ClientContext context, String tserver, Map<KeyExtent, List<Range>> tabletsRanges, TabletLocator parent) throws AccumuloSecurityException, AccumuloException {
    final TreeMap<Key, Value> results = new TreeMap<>();
    ResultReceiver rr = entries -> {
        for (Entry<Key, Value> entry : entries) {
            try {
                results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    };
    ScannerOptions opts = null;
    try (SettableScannerOptions unsetOpts = new SettableScannerOptions()) {
        opts = unsetOpts.setColumns(locCols);
    }
    Map<KeyExtent, List<Range>> unscanned = new HashMap<>();
    Map<KeyExtent, List<Range>> failures = new HashMap<>();
    try {
        TabletServerBatchReaderIterator.doLookup(context, tserver, tabletsRanges, failures, unscanned, rr, columns, opts, Authorizations.EMPTY);
        if (!failures.isEmpty()) {
            // invalidate extents in parents cache
            if (log.isTraceEnabled())
                log.trace("lookupTablets failed for {} extents", failures.size());
            parent.invalidateCache(failures.keySet());
        }
    } catch (IOException e) {
        log.trace("lookupTablets failed server={}", tserver, e);
        parent.invalidateCache(context, tserver);
    } catch (AccumuloServerException e) {
        log.trace("lookupTablets failed server={}", tserver, e);
        throw e;
    }
    return MetadataLocationObtainer.getMetadataLocationEntries(results).getLocations();
}

Example 3 with TabletLocation

use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.

the class TabletLocatorImplTest method createMetaCacheKE.

static TreeMap<KeyExtent, TabletLocation> createMetaCacheKE(Object... data) {
    TreeMap<KeyExtent, TabletLocation> mcke = new TreeMap<>();
    for (int i = 0; i < data.length; i += 2) {
        KeyExtent ke = (KeyExtent) data[i];
        String loc = (String) data[i + 1];
        mcke.put(ke, new TabletLocation(ke, loc, "1"));
    }
    return mcke;
}

Example 4 with TabletLocation

use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.

the class BulkImporter method findOverlappingTablets.

public static List<TabletLocation> findOverlappingTablets(ServerContext context, VolumeManager vm, TabletLocator locator, Path file, Text startRow, Text endRow) throws Exception {
    List<TabletLocation> result = new ArrayList<>();
    Collection<ByteSequence> columnFamilies = Collections.emptyList();
    String filename = file.toString();
    // log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
    FileSystem fs = vm.getFileSystemByPath(file);
    try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder().forFile(filename, fs, fs.getConf(), context.getCryptoService()).withTableConfiguration(context.getConfiguration()).seekToBeginning().build()) {
        Text row = startRow;
        if (row == null)
            row = new Text();
        while (true) {
            // log.debug(filename + " Seeking to row " + row);
            reader.seek(new Range(row, null), columnFamilies, false);
            if (!reader.hasTop()) {
                // log.debug(filename + " not found");
                break;
            }
            row = reader.getTopKey().getRow();
            TabletLocation tabletLocation = locator.locateTablet(context, row, false, true);
            // log.debug(filename + " found row " + row + " at location " + tabletLocation);
            result.add(tabletLocation);
            row = tabletLocation.tablet_extent.endRow();
            if (row != null && (endRow == null || row.compareTo(endRow) < 0)) {
                row = new Text(row);
                row.append(byte0, 0, byte0.length);
            } else
                break;
        }
    }
    // log.debug(filename + " to be sent to " + result);
    return result;
}

Example 5 with TabletLocation

use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.

the class BulkImporterTest method testFindOverlappingTablets.

@Test
public void testFindOverlappingTablets() throws Exception {
    MockTabletLocator locator = new MockTabletLocator();
    FileSystem fs = FileSystem.getLocal(new Configuration());
    ServerContext context = MockServerContext.get();
    EasyMock.replay(context);
    String file = "target/testFile.rf";
    fs.delete(new Path(file), true);
    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(file, fs, fs.getConf(), CryptoServiceFactory.newDefaultInstance()).withTableConfiguration(context.getConfiguration()).build();
    writer.startDefaultLocalityGroup();
    Value empty = new Value();
    writer.append(new Key("a", "cf", "cq"), empty);
    writer.append(new Key("a", "cf", "cq1"), empty);
    writer.append(new Key("a", "cf", "cq2"), empty);
    writer.append(new Key("a", "cf", "cq3"), empty);
    writer.append(new Key("a", "cf", "cq4"), empty);
    writer.append(new Key("a", "cf", "cq5"), empty);
    writer.append(new Key("d", "cf", "cq"), empty);
    writer.append(new Key("d", "cf", "cq1"), empty);
    writer.append(new Key("d", "cf", "cq2"), empty);
    writer.append(new Key("d", "cf", "cq3"), empty);
    writer.append(new Key("d", "cf", "cq4"), empty);
    writer.append(new Key("d", "cf", "cq5"), empty);
    writer.append(new Key("dd", "cf", "cq1"), empty);
    writer.append(new Key("ichabod", "cf", "cq"), empty);
    writer.append(new Key("icky", "cf", "cq1"), empty);
    writer.append(new Key("iffy", "cf", "cq2"), empty);
    writer.append(new Key("internal", "cf", "cq3"), empty);
    writer.append(new Key("is", "cf", "cq4"), empty);
    writer.append(new Key("iterator", "cf", "cq5"), empty);
    writer.append(new Key("xyzzy", "cf", "cq"), empty);
    writer.close();
    try (var vm = VolumeManagerImpl.getLocalForTesting("file:///")) {
        List<TabletLocation> overlaps = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file));
        assertEquals(5, overlaps.size());
        Collections.sort(overlaps);
        assertEquals(new KeyExtent(tableId, new Text("a"), null), overlaps.get(0).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps.get(1).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps.get(2).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps.get(3).tablet_extent);
        assertEquals(new KeyExtent(tableId, null, new Text("l")), overlaps.get(4).tablet_extent);
        List<TabletLocation> overlaps2 = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file), new KeyExtent(tableId, new Text("h"), new Text("b")));
        assertEquals(3, overlaps2.size());
        assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps2.get(0).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps2.get(1).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps2.get(2).tablet_extent);
        assertEquals(locator.invalidated, 1);
    }
}