Examples with KeyExtent org.apache.accumulo.core.data.impl.KeyExtent used on opensource projects

Example 86 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletServerBatchReaderIterator method binRanges.

private void binRanges(TabletLocator tabletLocator, List<Range> ranges, Map<String, Map<KeyExtent, List<Range>>> binnedRanges) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
    int lastFailureSize = Integer.MAX_VALUE;
    while (true) {
        binnedRanges.clear();
        List<Range> failures = tabletLocator.binRanges(context, ranges, binnedRanges);
        if (failures.size() > 0) {
            // need to always do the check when failures occur
            if (failures.size() >= lastFailureSize)
                if (!Tables.exists(instance, tableId))
                    throw new TableDeletedException(tableId.canonicalID());
                else if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
                    throw new TableOfflineException(instance, tableId.canonicalID());
            lastFailureSize = failures.size();
            if (log.isTraceEnabled())
                log.trace("Failed to bin {} ranges, tablet locations were null, retrying in 100ms", failures.size());
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        } else {
            break;
        }
    }
    // truncate the ranges to within the tablets... this makes it easier to know what work
    // needs to be redone when failures occurs and tablets have merged or split
    Map<String, Map<KeyExtent, List<Range>>> binnedRanges2 = new HashMap<>();
    for (Entry<String, Map<KeyExtent, List<Range>>> entry : binnedRanges.entrySet()) {
        Map<KeyExtent, List<Range>> tabletMap = new HashMap<>();
        binnedRanges2.put(entry.getKey(), tabletMap);
        for (Entry<KeyExtent, List<Range>> tabletRanges : entry.getValue().entrySet()) {
            Range tabletRange = tabletRanges.getKey().toDataRange();
            List<Range> clippedRanges = new ArrayList<>();
            tabletMap.put(tabletRanges.getKey(), clippedRanges);
            for (Range range : tabletRanges.getValue()) clippedRanges.add(tabletRange.clip(range));
        }
    }
    binnedRanges.clear();
    binnedRanges.putAll(binnedRanges2);
}

Example 87 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletServerBatchWriter method updateAuthorizationFailures.

private void updateAuthorizationFailures(Map<KeyExtent, SecurityErrorCode> authorizationFailures) {
    if (authorizationFailures.size() > 0) {
        // was a table deleted?
        HashSet<Table.ID> tableIds = new HashSet<>();
        for (KeyExtent ke : authorizationFailures.keySet()) tableIds.add(ke.getTableId());
        Tables.clearCache(context.getInstance());
        for (Table.ID tableId : tableIds) if (!Tables.exists(context.getInstance(), tableId))
            throw new TableDeletedException(tableId.canonicalID());
        synchronized (this) {
            somethingFailed = true;
            mergeAuthorizationFailures(this.authorizationFailures, authorizationFailures);
            this.notifyAll();
        }
    }
}

Example 88 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class TabletServerBatchWriter method updateAuthorizationFailures.

private void updateAuthorizationFailures(Set<KeyExtent> keySet, SecurityErrorCode code) {
    HashMap<KeyExtent, SecurityErrorCode> map = new HashMap<>();
    for (KeyExtent ke : keySet) map.put(ke, code);
    updateAuthorizationFailures(map);
}

Example 89 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class OfflineIterator method nextTablet.

private void nextTablet() throws TableNotFoundException, AccumuloException, IOException {
    Range nextRange = null;
    if (currentExtent == null) {
        Text startRow;
        if (range.getStartKey() != null)
            startRow = range.getStartKey().getRow();
        else
            startRow = new Text();
        nextRange = new Range(new KeyExtent(tableId, startRow, null).getMetadataEntry(), true, null, false);
    } else {
        if (currentExtent.getEndRow() == null) {
            iter = null;
            return;
        }
        if (range.afterEndKey(new Key(currentExtent.getEndRow()).followingKey(PartialKey.ROW))) {
            iter = null;
            return;
        }
        nextRange = new Range(currentExtent.getMetadataEntry(), false, null, false);
    }
    List<String> relFiles = new ArrayList<>();
    Pair<KeyExtent, String> eloc = getTabletFiles(nextRange, relFiles);
    while (eloc.getSecond() != null) {
        if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
            Tables.clearCache(instance);
            if (Tables.getTableState(instance, tableId) != TableState.OFFLINE) {
                throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + eloc.getFirst());
            }
        }
        sleepUninterruptibly(250, TimeUnit.MILLISECONDS);
        eloc = getTabletFiles(nextRange, relFiles);
    }
    KeyExtent extent = eloc.getFirst();
    if (!extent.getTableId().equals(tableId)) {
        throw new AccumuloException(" did not find tablets for table " + tableId + " " + extent);
    }
    if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
        throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
    // Old property is only used to resolve relative paths into absolute paths. For systems upgraded
    // with relative paths, it's assumed that correct instance.dfs.{uri,dir} is still correct in the configuration
    @SuppressWarnings("deprecation") String tablesDir = config.get(Property.INSTANCE_DFS_DIR) + Constants.HDFS_TABLES_DIR;
    List<String> absFiles = new ArrayList<>();
    for (String relPath : relFiles) {
        if (relPath.contains(":")) {
            absFiles.add(relPath);
        } else {
            // handle old-style relative paths
            if (relPath.startsWith("..")) {
                absFiles.add(tablesDir + relPath.substring(2));
            } else {
                absFiles.add(tablesDir + "/" + tableId + relPath);
            }
        }
    }
    iter = createIterator(extent, absFiles);
    iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() == 0 ? false : true);
    currentExtent = extent;
}

Example 90 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class AbstractInputFormat method getSplits.

/**
 * Gets the splits of the tables that have been set on the job by reading the metadata table for the specified ranges.
 *
 * @return the splits from the tables based on the ranges.
 * @throws java.io.IOException
 *           if a table set on the job doesn't exist or an error occurs initializing the tablet locator
 */
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException {
    Level logLevel = getLogLevel(context);
    log.setLevel(logLevel);
    validateOptions(context);
    Random random = new Random();
    LinkedList<InputSplit> splits = new LinkedList<>();
    Map<String, InputTableConfig> tableConfigs = getInputTableConfigs(context);
    for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
        String tableName = tableConfigEntry.getKey();
        InputTableConfig tableConfig = tableConfigEntry.getValue();
        Instance instance = getInstance(context);
        Table.ID tableId;
        // resolve table name to id once, and use id from this point forward
        if (DeprecationUtil.isMockInstance(instance)) {
            tableId = Table.ID.of("");
        } else {
            try {
                tableId = Tables.getTableId(instance, tableName);
            } catch (TableNotFoundException e) {
                throw new IOException(e);
            }
        }
        Authorizations auths = getScanAuthorizations(context);
        String principal = getPrincipal(context);
        AuthenticationToken token = getAuthenticationToken(context);
        boolean batchScan = InputConfigurator.isBatchScan(CLASS, context.getConfiguration());
        boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
        if (batchScan && !supportBatchScan)
            throw new IllegalArgumentException("BatchScanner optimization not available for offline scan, isolated, or local iterators");
        boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
        if (batchScan && !autoAdjust)
            throw new IllegalArgumentException("AutoAdjustRanges must be enabled when using BatchScanner optimization");
        List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
        if (ranges.isEmpty()) {
            ranges = new ArrayList<>(1);
            ranges.add(new Range());
        }
        // get the metadata information for these ranges
        Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
        TabletLocator tl;
        try {
            if (tableConfig.isOfflineScan()) {
                binnedRanges = binOfflineTable(context, tableId, ranges);
                while (binnedRanges == null) {
                    // Some tablets were still online, try again
                    // sleep randomly between 100 and 200 ms
                    sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                    binnedRanges = binOfflineTable(context, tableId, ranges);
                }
            } else {
                tl = InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), tableId);
                // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
                tl.invalidateCache();
                ClientContext clientContext = new ClientContext(getInstance(context), new Credentials(getPrincipal(context), getAuthenticationToken(context)), getClientConfiguration(context));
                while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
                    if (!DeprecationUtil.isMockInstance(instance)) {
                        String tableIdStr = tableId.canonicalID();
                        if (!Tables.exists(instance, tableId))
                            throw new TableDeletedException(tableIdStr);
                        if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
                            throw new TableOfflineException(instance, tableIdStr);
                    }
                    binnedRanges.clear();
                    log.warn("Unable to locate bins for specified ranges. Retrying.");
                    // sleep randomly between 100 and 200 ms
                    sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                    tl.invalidateCache();
                }
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
        // all of this code will add either range per each locations or split ranges and add range-location split
        // Map from Range to Array of Locations, we only use this if we're don't split
        HashMap<Range, ArrayList<String>> splitsToAdd = null;
        if (!autoAdjust)
            splitsToAdd = new HashMap<>();
        HashMap<String, String> hostNameCache = new HashMap<>();
        for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
            String ip = tserverBin.getKey().split(":", 2)[0];
            String location = hostNameCache.get(ip);
            if (location == null) {
                InetAddress inetAddress = InetAddress.getByName(ip);
                location = inetAddress.getCanonicalHostName();
                hostNameCache.put(ip, location);
            }
            for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
                Range ke = extentRanges.getKey().toDataRange();
                if (batchScan) {
                    // group ranges by tablet to be read by a BatchScanner
                    ArrayList<Range> clippedRanges = new ArrayList<>();
                    for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));
                    BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location });
                    SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                    splits.add(split);
                } else {
                    // not grouping by tablet
                    for (Range r : extentRanges.getValue()) {
                        if (autoAdjust) {
                            // divide ranges into smaller ranges, based on the tablets
                            RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(), ke.clip(r), new String[] { location });
                            SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                            split.setOffline(tableConfig.isOfflineScan());
                            split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                            split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                            splits.add(split);
                        } else {
                            // don't divide ranges
                            ArrayList<String> locations = splitsToAdd.get(r);
                            if (locations == null)
                                locations = new ArrayList<>(1);
                            locations.add(location);
                            splitsToAdd.put(r, locations);
                        }
                    }
                }
            }
        }
        if (!autoAdjust)
            for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
                RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(), entry.getKey(), entry.getValue().toArray(new String[0]));
                SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                split.setOffline(tableConfig.isOfflineScan());
                split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                splits.add(split);
            }
    }
    return splits;
}