Examples with TabletServerId org.apache.accumulo.core.spi.balancer.data.TabletServerId used on opensource projects

Example 16 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class ChaoticLoadBalancerTest method testUnevenAssignment.

@Test
public void testUnevenAssignment() {
    servers.clear();
    for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
        String cString = Character.toString(c);
        TabletServerId tsi = new TabletServerIdImpl("127.0.0.1", c, cString);
        FakeTServer fakeTServer = new FakeTServer();
        servers.put(tsi, fakeTServer);
        fakeTServer.tablets.add(makeTablet(cString, null, null));
    }
    // Put more tablets on one server, but not more than the number of servers
    Entry<TabletServerId, FakeTServer> first = servers.entrySet().iterator().next();
    first.getValue().tablets.add(makeTablet("newTable", "a", null));
    first.getValue().tablets.add(makeTablet("newTable", "b", "a"));
    first.getValue().tablets.add(makeTablet("newTable", "c", "b"));
    first.getValue().tablets.add(makeTablet("newTable", "d", "c"));
    first.getValue().tablets.add(makeTablet("newTable", "e", "d"));
    first.getValue().tablets.add(makeTablet("newTable", "f", "e"));
    first.getValue().tablets.add(makeTablet("newTable", "g", "f"));
    first.getValue().tablets.add(makeTablet("newTable", "h", "g"));
    first.getValue().tablets.add(makeTablet("newTable", "i", null));
    TestChaoticLoadBalancer balancer = new TestChaoticLoadBalancer();
    Set<TabletId> migrations = Collections.emptySet();
    // Just want to make sure it gets some migrations, randomness prevents guarantee of a defined
    // amount, or even expected amount
    List<TabletMigration> migrationsOut = new ArrayList<>();
    while (!migrationsOut.isEmpty()) {
        balancer.balance(new BalanceParamsImpl(getAssignments(servers), migrations, migrationsOut));
    }
}

Example 17 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class BalanceParamsImpl method addMigration.

public void addMigration(KeyExtent extent, TServerInstance oldServer, TServerInstance newServer) {
    TabletId id = new TabletIdImpl(extent);
    TabletServerId oldTsid = new TabletServerIdImpl(oldServer);
    TabletServerId newTsid = new TabletServerIdImpl(newServer);
    migrationsOut.add(new TabletMigration(id, oldTsid, newTsid));
}

Example 18 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class HostRegexTableLoadBalancer method getAssignments.

@Override
public void getAssignments(AssignmentParameters params) {
    Map<String, SortedMap<TabletServerId, TServerStatus>> pools = splitCurrentByRegex(params.currentStatus());
    // group the unassigned into tables
    Map<TableId, Map<TabletId, TabletServerId>> groupedUnassigned = new HashMap<>();
    params.unassignedTablets().forEach((ke, lastTserver) -> groupedUnassigned.computeIfAbsent(ke.getTable(), k -> new HashMap<>()).put(ke, lastTserver));
    Map<TableId, String> tableIdToTableName = environment.getTableIdMap().entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    // Send a view of the current servers to the tables tablet balancer
    for (Entry<TableId, Map<TabletId, TabletServerId>> e : groupedUnassigned.entrySet()) {
        Map<TabletId, TabletServerId> newAssignments = new HashMap<>();
        String tableName = tableIdToTableName.get(e.getKey());
        String poolName = getPoolNameForTable(tableName);
        SortedMap<TabletServerId, TServerStatus> currentView = pools.get(poolName);
        if (currentView == null || currentView.isEmpty()) {
            LOG.warn("No tablet servers online for table {}, assigning within default pool", tableName);
            currentView = pools.get(DEFAULT_POOL);
            if (currentView == null) {
                LOG.error("No tablet servers exist in the default pool, unable to assign tablets for table {}", tableName);
                continue;
            }
        }
        LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers {}", e.getValue().size(), tableName, currentView.keySet());
        getBalancerForTable(e.getKey()).getAssignments(new AssignmentParamsImpl(currentView, e.getValue(), newAssignments));
        newAssignments.forEach(params::addAssignment);
    }
}

Example 19 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class HostRegexTableLoadBalancer method balance.

@Override
public long balance(BalanceParameters params) {
    long minBalanceTime = 20_000;
    // Iterate over the tables and balance each of them
    Map<String, TableId> tableIdMap = environment.getTableIdMap();
    Map<TableId, String> tableIdToTableName = tableIdMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    tableIdToTableName.keySet().forEach(this::checkTableConfig);
    long now = System.currentTimeMillis();
    HrtlbConf myConf = hrtlbConf.get();
    SortedMap<TabletServerId, TServerStatus> current = params.currentStatus();
    Set<TabletId> migrations = params.currentMigrations();
    List<TabletMigration> migrationsOut = params.migrationsOut();
    Map<String, SortedMap<TabletServerId, TServerStatus>> currentGrouped = splitCurrentByRegex(params.currentStatus());
    if ((now - this.lastOOBCheck) > myConf.oobCheckMillis) {
        try {
            // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
            for (String table : tableIdMap.keySet()) {
                LOG.debug("Checking for out of bounds tablets for table {}", table);
                String tablePoolName = getPoolNameForTable(table);
                for (Entry<TabletServerId, TServerStatus> e : current.entrySet()) {
                    // pool names are the same as table names, except in the DEFAULT case.
                    // If this table is assigned to a pool for this host, then move on.
                    List<String> hostPools = getPoolNamesForHost(e.getKey());
                    if (hostPools.contains(tablePoolName)) {
                        continue;
                    }
                    TableId tid = tableIdMap.get(table);
                    if (tid == null) {
                        LOG.warn("Unable to check for out of bounds tablets for table {}," + " it may have been deleted or renamed.", table);
                        continue;
                    }
                    try {
                        List<TabletStatistics> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
                        if (outOfBoundsTablets == null) {
                            continue;
                        }
                        for (TabletStatistics ts : outOfBoundsTablets) {
                            if (migrations.contains(ts.getTabletId())) {
                                LOG.debug("Migration for out of bounds tablet {} has already been requested", ts.getTabletId());
                                continue;
                            }
                            String poolName = getPoolNameForTable(table);
                            SortedMap<TabletServerId, TServerStatus> currentView = currentGrouped.get(poolName);
                            if (currentView != null) {
                                int skip = random.nextInt(currentView.size());
                                Iterator<TabletServerId> iter = currentView.keySet().iterator();
                                for (int i = 0; i < skip; i++) {
                                    iter.next();
                                }
                                TabletServerId nextTS = iter.next();
                                LOG.info("Tablet {} is currently outside the bounds of the" + " regex, migrating from {} to {}", ts.getTabletId(), e.getKey(), nextTS);
                                migrationsOut.add(new TabletMigration(ts.getTabletId(), e.getKey(), nextTS));
                                if (migrationsOut.size() >= myConf.maxTServerMigrations) {
                                    break;
                                }
                            } else {
                                LOG.warn("No tablet servers online for pool {}, unable to" + " migrate out of bounds tablets", poolName);
                            }
                        }
                    } catch (AccumuloException | AccumuloSecurityException e1) {
                        LOG.error("Error in OOB check getting tablets for table {} from server {} {}", tid, e.getKey().getHost(), e);
                    }
                }
            }
        } finally {
            // this could have taken a while...get a new time
            this.lastOOBCheck = System.currentTimeMillis();
        }
    }
    if (!migrationsOut.isEmpty()) {
        LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
        LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
        return minBalanceTime;
    }
    if (migrations != null && !migrations.isEmpty()) {
        if (migrations.size() >= myConf.maxOutstandingMigrations) {
            LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
            if (LOG.isTraceEnabled()) {
                LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
            }
            return minBalanceTime;
        }
        LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
        }
        migrationsFromLastPass.keySet().retainAll(migrations);
        SortedMap<TabletServerId, TServerStatusImpl> currentCopy = new TreeMap<>();
        current.forEach((tid, status) -> currentCopy.put(tid, (TServerStatusImpl) status));
        Multimap<TabletServerId, String> serverTableIdCopied = HashMultimap.create();
        for (TabletMigration migration : migrationsFromLastPass.values()) {
            TableStatisticsImpl fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.getTablet().getTable().canonical(), migration.getOldTabletServer());
            if (fromInfo != null) {
                fromInfo.setOnlineTabletCount(fromInfo.getOnlineTabletCount() - 1);
            }
            TableStatisticsImpl toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.getTablet().getTable().canonical(), migration.getNewTabletServer());
            if (toInfo != null) {
                toInfo.setOnlineTabletCount(toInfo.getOnlineTabletCount() + 1);
            }
        }
        migrations = EMPTY_MIGRATIONS;
    } else {
        migrationsFromLastPass.clear();
    }
    for (TableId tableId : tableIdMap.values()) {
        String tableName = tableIdToTableName.get(tableId);
        String regexTableName = getPoolNameForTable(tableName);
        SortedMap<TabletServerId, TServerStatus> currentView = currentGrouped.get(regexTableName);
        if (currentView == null) {
            LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
            continue;
        }
        ArrayList<TabletMigration> newMigrations = new ArrayList<>();
        getBalancerForTable(tableId).balance(new BalanceParamsImpl(currentView, migrations, newMigrations));
        if (newMigrations.isEmpty()) {
            tableToTimeSinceNoMigrations.remove(tableId);
        } else if (tableToTimeSinceNoMigrations.containsKey(tableId)) {
            if ((now - tableToTimeSinceNoMigrations.get(tableId)) > HOURS.toMillis(1)) {
                LOG.warn("We have been consistently producing migrations for {}: {}", tableName, Iterables.limit(newMigrations, 10));
            }
        } else {
            tableToTimeSinceNoMigrations.put(tableId, now);
        }
        migrationsOut.addAll(newMigrations);
        if (migrationsOut.size() >= myConf.maxTServerMigrations) {
            break;
        }
    }
    for (TabletMigration migration : migrationsOut) {
        migrationsFromLastPass.put(migration.getTablet(), migration);
    }
    LOG.info("Migrating tablets for balance: {}", migrationsOut);
    return minBalanceTime;
}

Example 20 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class SimpleLoadBalancer method getAssignment.

public TabletServerId getAssignment(SortedMap<TabletServerId, TServerStatus> locations, TabletServerId last) {
    if (locations.isEmpty())
        return null;
    if (last != null) {
        // Maintain locality
        String fakeSessionID = " ";
        TabletServerId simple = new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID);
        Iterator<TabletServerId> find = locations.tailMap(simple).keySet().iterator();
        if (find.hasNext()) {
            TabletServerId current = find.next();
            if (current.getHost().equals(last.getHost()))
                return current;
        }
    }
    // in the current list.
    if (assignments == null || !assignments.hasNext())
        assignments = randomize(locations.keySet()).iterator();
    TabletServerId result = assignments.next();
    if (!locations.containsKey(result)) {
        assignments = null;
        return randomize(locations.keySet()).iterator().next();
    }
    return result;
}