Examples with KeyExtent org.apache.accumulo.core.data.impl.KeyExtent used on opensource projects

Example 51 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class ChaoticLoadBalancer method getAssignments.

@Override
public void getAssignments(SortedMap<TServerInstance, TabletServerStatus> current, Map<KeyExtent, TServerInstance> unassigned, Map<KeyExtent, TServerInstance> assignments) {
    long total = assignments.size() + unassigned.size();
    long avg = (long) Math.ceil(((double) total) / current.size());
    Map<TServerInstance, Long> toAssign = new HashMap<>();
    List<TServerInstance> tServerArray = new ArrayList<>();
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        long numTablets = 0;
        for (TableInfo ti : e.getValue().getTableMap().values()) {
            numTablets += ti.tablets;
        }
        if (numTablets <= avg) {
            tServerArray.add(e.getKey());
            toAssign.put(e.getKey(), avg - numTablets);
        }
    }
    if (tServerArray.isEmpty()) {
        // No tservers to assign to
        return;
    }
    for (KeyExtent ke : unassigned.keySet()) {
        int index = r.nextInt(tServerArray.size());
        TServerInstance dest = tServerArray.get(index);
        assignments.put(ke, dest);
        long remaining = toAssign.get(dest) - 1;
        if (remaining == 0) {
            tServerArray.remove(index);
            toAssign.remove(dest);
        } else {
            toAssign.put(dest, remaining);
        }
    }
}

Example 52 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class DefaultLoadBalancer method selectTablet.

static KeyExtent selectTablet(TServerInstance tserver, Map<KeyExtent, TabletStats> extents) {
    if (extents.size() == 0)
        return null;
    KeyExtent mostRecentlySplit = null;
    long splitTime = 0;
    for (Entry<KeyExtent, TabletStats> entry : extents.entrySet()) if (entry.getValue().splitCreationTime >= splitTime) {
        splitTime = entry.getValue().splitCreationTime;
        mostRecentlySplit = entry.getKey();
    }
    return mostRecentlySplit;
}

Example 53 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class DefaultLoadBalancer method move.

/**
 * Select a tablet based on differences between table loads; if the loads are even, use the busiest table
 */
List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count, Map<Table.ID, Map<KeyExtent, TabletStats>> donerTabletStats) {
    List<TabletMigration> result = new ArrayList<>();
    if (count == 0)
        return result;
    // Copy counts so we can update them as we propose migrations
    Map<Table.ID, Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
    Map<Table.ID, Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
    for (int i = 0; i < count; i++) {
        Table.ID table;
        Integer tooLittleCount;
        if (tableToBalance == null) {
            // find a table to migrate
            // look for an uneven table count
            int biggestDifference = 0;
            Table.ID biggestDifferenceTable = null;
            for (Entry<Table.ID, Integer> tableEntry : tooMuchMap.entrySet()) {
                Table.ID tableID = tableEntry.getKey();
                if (tooLittleMap.get(tableID) == null)
                    tooLittleMap.put(tableID, 0);
                int diff = tableEntry.getValue() - tooLittleMap.get(tableID);
                if (diff > biggestDifference) {
                    biggestDifference = diff;
                    biggestDifferenceTable = tableID;
                }
            }
            if (biggestDifference < 2) {
                table = busiest(tooMuch.status.tableMap);
            } else {
                table = biggestDifferenceTable;
            }
        } else {
            // just balance the given table
            table = tableToBalance;
        }
        Map<KeyExtent, TabletStats> onlineTabletsForTable = donerTabletStats.get(table);
        try {
            if (onlineTabletsForTable == null) {
                onlineTabletsForTable = new HashMap<>();
                List<TabletStats> stats = getOnlineTabletsForTable(tooMuch.server, table);
                if (null == stats) {
                    log.warn("Unable to find tablets to move");
                    return result;
                }
                for (TabletStats stat : stats) onlineTabletsForTable.put(new KeyExtent(stat.extent), stat);
                donerTabletStats.put(table, onlineTabletsForTable);
            }
        } catch (Exception ex) {
            log.error("Unable to select a tablet to move", ex);
            return result;
        }
        KeyExtent extent = selectTablet(tooMuch.server, onlineTabletsForTable);
        onlineTabletsForTable.remove(extent);
        if (extent == null)
            return result;
        tooMuchMap.put(table, tooMuchMap.get(table) - 1);
        /**
         * If a table grows from 1 tablet then tooLittleMap.get(table) can return a null, since there is only one tabletserver that holds all of the tablets. Here
         * we check to see if in fact that is the case and if so set the value to 0.
         */
        tooLittleCount = tooLittleMap.get(table);
        if (tooLittleCount == null) {
            tooLittleCount = 0;
        }
        tooLittleMap.put(table, tooLittleCount + 1);
        tooMuch.count--;
        tooLittle.count++;
        result.add(new TabletMigration(extent, tooMuch.server, tooLittle.server));
    }
    return result;
}

Example 54 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class GroupBalancer method getAssignments.

@Override
public void getAssignments(SortedMap<TServerInstance, TabletServerStatus> current, Map<KeyExtent, TServerInstance> unassigned, Map<KeyExtent, TServerInstance> assignments) {
    if (current.size() == 0) {
        return;
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    List<ComparablePair<String, KeyExtent>> tabletsByGroup = new ArrayList<>();
    for (Entry<KeyExtent, TServerInstance> entry : unassigned.entrySet()) {
        TServerInstance last = entry.getValue();
        if (last != null) {
            // Maintain locality
            String fakeSessionID = " ";
            TServerInstance simple = new TServerInstance(last.getLocation(), fakeSessionID);
            Iterator<TServerInstance> find = current.tailMap(simple).keySet().iterator();
            if (find.hasNext()) {
                TServerInstance tserver = find.next();
                if (tserver.host().equals(last.host())) {
                    assignments.put(entry.getKey(), tserver);
                    continue;
                }
            }
        }
        tabletsByGroup.add(new ComparablePair<>(partitioner.apply(entry.getKey()), entry.getKey()));
    }
    Collections.sort(tabletsByGroup);
    Iterator<TServerInstance> tserverIter = Iterators.cycle(current.keySet());
    for (ComparablePair<String, KeyExtent> pair : tabletsByGroup) {
        KeyExtent ke = pair.getSecond();
        assignments.put(ke, tserverIter.next());
    }
}

Example 55 with KeyExtent

use of org.apache.accumulo.core.data.impl.KeyExtent in project accumulo by apache.

the class GroupBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    if (!shouldBalance(current, migrations)) {
        return 5000;
    }
    if (System.currentTimeMillis() - lastRun < getWaitTime()) {
        return 5000;
    }
    MapCounter<String> groupCounts = new MapCounter<>();
    Map<TServerInstance, TserverGroupInfo> tservers = new HashMap<>();
    for (TServerInstance tsi : current.keySet()) {
        tservers.put(tsi, new TserverGroupInfo(tsi));
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    // collect stats about current state
    for (Pair<KeyExtent, Location> entry : getLocationProvider()) {
        String group = partitioner.apply(entry.getFirst());
        Location loc = entry.getSecond();
        if (loc.equals(Location.NONE) || !tservers.containsKey(loc.getTserverInstance())) {
            return 5000;
        }
        groupCounts.increment(group, 1);
        TserverGroupInfo tgi = tservers.get(loc.getTserverInstance());
        tgi.addGroup(group);
    }
    Map<String, Integer> expectedCounts = new HashMap<>();
    int totalExtra = 0;
    for (String group : groupCounts.keySet()) {
        long groupCount = groupCounts.get(group);
        totalExtra += groupCount % current.size();
        expectedCounts.put(group, (int) (groupCount / current.size()));
    }
    // The number of extra tablets from all groups that each tserver must have.
    int expectedExtra = totalExtra / current.size();
    int maxExtraGroups = expectedExtra + 1;
    expectedCounts = Collections.unmodifiableMap(expectedCounts);
    tservers = Collections.unmodifiableMap(tservers);
    for (TserverGroupInfo tgi : tservers.values()) {
        tgi.finishedAdding(expectedCounts);
    }
    Moves moves = new Moves();
    // The order of the following steps is important, because as ordered each step should not move any tablets moved by a previous step.
    balanceExpected(tservers, moves);
    if (moves.size() < getMaxMigrations()) {
        balanceExtraExpected(tservers, expectedExtra, moves);
        if (moves.size() < getMaxMigrations()) {
            boolean cont = balanceExtraMultiple(tservers, maxExtraGroups, moves);
            if (cont && moves.size() < getMaxMigrations()) {
                balanceExtraExtra(tservers, maxExtraGroups, moves);
            }
        }
    }
    populateMigrations(tservers.keySet(), migrationsOut, moves);
    lastRun = System.currentTimeMillis();
    return 5000;
}