Examples with TServerStatus org.apache.accumulo.core.spi.balancer.data.TServerStatus used on opensource projects

Example 6 with TServerStatus

use of org.apache.accumulo.core.spi.balancer.data.TServerStatus in project accumulo by apache.

the class SimpleLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.2", 1234, "b"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.3", 1234, "c"), new FakeTServer());
    List<TabletId> metadataTable = new ArrayList<>();
    String table = "t1";
    metadataTable.add(makeTablet(table, null, null));
    table = "t2";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, null, "a"));
    table = "t3";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, "b", "a"));
    metadataTable.add(makeTablet(table, "c", "b"));
    metadataTable.add(makeTablet(table, "d", "c"));
    metadataTable.add(makeTablet(table, "e", "d"));
    metadataTable.add(makeTablet(table, null, "e"));
    Collections.sort(metadataTable);
    TestSimpleLoadBalancer balancer = new TestSimpleLoadBalancer();
    SortedMap<TabletServerId, TServerStatus> current = new TreeMap<>();
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    assignTablets(metadataTable, servers, current, balancer);
    // Verify that the counts on the tables are correct
    Map<String, Integer> expectedCounts = new HashMap<>();
    expectedCounts.put("t1", 1);
    expectedCounts.put("t2", 1);
    expectedCounts.put("t3", 2);
    checkBalance(metadataTable, servers, expectedCounts);
    // Rebalance once
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    // Nothing should happen, we are balanced
    ArrayList<TabletMigration> out = new ArrayList<>();
    balancer.getMigrations(current, out);
    assertEquals(out.size(), 0);
    // Take down a tabletServer
    TabletServerId first = current.keySet().iterator().next();
    current.remove(first);
    FakeTServer remove = servers.remove(first);
    // reassign offline extents
    assignTablets(remove.tablets, servers, current, balancer);
    checkBalance(metadataTable, servers, null);
}

Example 7 with TServerStatus

use of org.apache.accumulo.core.spi.balancer.data.TServerStatus in project accumulo by apache.

the class HostRegexTableLoadBalancer method getAssignments.

@Override
public void getAssignments(AssignmentParameters params) {
    Map<String, SortedMap<TabletServerId, TServerStatus>> pools = splitCurrentByRegex(params.currentStatus());
    // group the unassigned into tables
    Map<TableId, Map<TabletId, TabletServerId>> groupedUnassigned = new HashMap<>();
    params.unassignedTablets().forEach((ke, lastTserver) -> groupedUnassigned.computeIfAbsent(ke.getTable(), k -> new HashMap<>()).put(ke, lastTserver));
    Map<TableId, String> tableIdToTableName = environment.getTableIdMap().entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    // Send a view of the current servers to the tables tablet balancer
    for (Entry<TableId, Map<TabletId, TabletServerId>> e : groupedUnassigned.entrySet()) {
        Map<TabletId, TabletServerId> newAssignments = new HashMap<>();
        String tableName = tableIdToTableName.get(e.getKey());
        String poolName = getPoolNameForTable(tableName);
        SortedMap<TabletServerId, TServerStatus> currentView = pools.get(poolName);
        if (currentView == null || currentView.isEmpty()) {
            LOG.warn("No tablet servers online for table {}, assigning within default pool", tableName);
            currentView = pools.get(DEFAULT_POOL);
            if (currentView == null) {
                LOG.error("No tablet servers exist in the default pool, unable to assign tablets for table {}", tableName);
                continue;
            }
        }
        LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers {}", e.getValue().size(), tableName, currentView.keySet());
        getBalancerForTable(e.getKey()).getAssignments(new AssignmentParamsImpl(currentView, e.getValue(), newAssignments));
        newAssignments.forEach(params::addAssignment);
    }
}

Example 8 with TServerStatus

use of org.apache.accumulo.core.spi.balancer.data.TServerStatus in project accumulo by apache.

the class HostRegexTableLoadBalancer method balance.

@Override
public long balance(BalanceParameters params) {
    long minBalanceTime = 20_000;
    // Iterate over the tables and balance each of them
    Map<String, TableId> tableIdMap = environment.getTableIdMap();
    Map<TableId, String> tableIdToTableName = tableIdMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
    tableIdToTableName.keySet().forEach(this::checkTableConfig);
    long now = System.currentTimeMillis();
    HrtlbConf myConf = hrtlbConf.get();
    SortedMap<TabletServerId, TServerStatus> current = params.currentStatus();
    Set<TabletId> migrations = params.currentMigrations();
    List<TabletMigration> migrationsOut = params.migrationsOut();
    Map<String, SortedMap<TabletServerId, TServerStatus>> currentGrouped = splitCurrentByRegex(params.currentStatus());
    if ((now - this.lastOOBCheck) > myConf.oobCheckMillis) {
        try {
            // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
            for (String table : tableIdMap.keySet()) {
                LOG.debug("Checking for out of bounds tablets for table {}", table);
                String tablePoolName = getPoolNameForTable(table);
                for (Entry<TabletServerId, TServerStatus> e : current.entrySet()) {
                    // pool names are the same as table names, except in the DEFAULT case.
                    // If this table is assigned to a pool for this host, then move on.
                    List<String> hostPools = getPoolNamesForHost(e.getKey());
                    if (hostPools.contains(tablePoolName)) {
                        continue;
                    }
                    TableId tid = tableIdMap.get(table);
                    if (tid == null) {
                        LOG.warn("Unable to check for out of bounds tablets for table {}," + " it may have been deleted or renamed.", table);
                        continue;
                    }
                    try {
                        List<TabletStatistics> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
                        if (outOfBoundsTablets == null) {
                            continue;
                        }
                        for (TabletStatistics ts : outOfBoundsTablets) {
                            if (migrations.contains(ts.getTabletId())) {
                                LOG.debug("Migration for out of bounds tablet {} has already been requested", ts.getTabletId());
                                continue;
                            }
                            String poolName = getPoolNameForTable(table);
                            SortedMap<TabletServerId, TServerStatus> currentView = currentGrouped.get(poolName);
                            if (currentView != null) {
                                int skip = random.nextInt(currentView.size());
                                Iterator<TabletServerId> iter = currentView.keySet().iterator();
                                for (int i = 0; i < skip; i++) {
                                    iter.next();
                                }
                                TabletServerId nextTS = iter.next();
                                LOG.info("Tablet {} is currently outside the bounds of the" + " regex, migrating from {} to {}", ts.getTabletId(), e.getKey(), nextTS);
                                migrationsOut.add(new TabletMigration(ts.getTabletId(), e.getKey(), nextTS));
                                if (migrationsOut.size() >= myConf.maxTServerMigrations) {
                                    break;
                                }
                            } else {
                                LOG.warn("No tablet servers online for pool {}, unable to" + " migrate out of bounds tablets", poolName);
                            }
                        }
                    } catch (AccumuloException | AccumuloSecurityException e1) {
                        LOG.error("Error in OOB check getting tablets for table {} from server {} {}", tid, e.getKey().getHost(), e);
                    }
                }
            }
        } finally {
            // this could have taken a while...get a new time
            this.lastOOBCheck = System.currentTimeMillis();
        }
    }
    if (!migrationsOut.isEmpty()) {
        LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
        LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
        return minBalanceTime;
    }
    if (migrations != null && !migrations.isEmpty()) {
        if (migrations.size() >= myConf.maxOutstandingMigrations) {
            LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
            if (LOG.isTraceEnabled()) {
                LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
            }
            return minBalanceTime;
        }
        LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
        }
        migrationsFromLastPass.keySet().retainAll(migrations);
        SortedMap<TabletServerId, TServerStatusImpl> currentCopy = new TreeMap<>();
        current.forEach((tid, status) -> currentCopy.put(tid, (TServerStatusImpl) status));
        Multimap<TabletServerId, String> serverTableIdCopied = HashMultimap.create();
        for (TabletMigration migration : migrationsFromLastPass.values()) {
            TableStatisticsImpl fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.getTablet().getTable().canonical(), migration.getOldTabletServer());
            if (fromInfo != null) {
                fromInfo.setOnlineTabletCount(fromInfo.getOnlineTabletCount() - 1);
            }
            TableStatisticsImpl toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.getTablet().getTable().canonical(), migration.getNewTabletServer());
            if (toInfo != null) {
                toInfo.setOnlineTabletCount(toInfo.getOnlineTabletCount() + 1);
            }
        }
        migrations = EMPTY_MIGRATIONS;
    } else {
        migrationsFromLastPass.clear();
    }
    for (TableId tableId : tableIdMap.values()) {
        String tableName = tableIdToTableName.get(tableId);
        String regexTableName = getPoolNameForTable(tableName);
        SortedMap<TabletServerId, TServerStatus> currentView = currentGrouped.get(regexTableName);
        if (currentView == null) {
            LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
            continue;
        }
        ArrayList<TabletMigration> newMigrations = new ArrayList<>();
        getBalancerForTable(tableId).balance(new BalanceParamsImpl(currentView, migrations, newMigrations));
        if (newMigrations.isEmpty()) {
            tableToTimeSinceNoMigrations.remove(tableId);
        } else if (tableToTimeSinceNoMigrations.containsKey(tableId)) {
            if ((now - tableToTimeSinceNoMigrations.get(tableId)) > HOURS.toMillis(1)) {
                LOG.warn("We have been consistently producing migrations for {}: {}", tableName, Iterables.limit(newMigrations, 10));
            }
        } else {
            tableToTimeSinceNoMigrations.put(tableId, now);
        }
        migrationsOut.addAll(newMigrations);
        if (migrationsOut.size() >= myConf.maxTServerMigrations) {
            break;
        }
    }
    for (TabletMigration migration : migrationsOut) {
        migrationsFromLastPass.put(migration.getTablet(), migration);
    }
    LOG.info("Migrating tablets for balance: {}", migrationsOut);
    return minBalanceTime;
}

Example 9 with TServerStatus

use of org.apache.accumulo.core.spi.balancer.data.TServerStatus in project accumulo by apache.

the class TableLoadBalancerTest method generateFakeTablets.

static List<TabletStatistics> generateFakeTablets(TabletServerId tserver, TableId tableId) {
    List<TabletStatistics> result = new ArrayList<>();
    TServerStatus tableInfo = state.get(tserver);
    // generate some fake tablets
    for (int i = 0; i < tableInfo.getTableMap().get(tableId.canonical()).getOnlineTabletCount(); i++) {
        TabletStats stats = new TabletStats();
        stats.extent = new KeyExtent(tableId, new Text(tserver.getHost() + String.format("%03d", i + 1)), new Text(tserver.getHost() + String.format("%03d", i))).toThrift();
        result.add(new TabletStatisticsImpl(stats));
    }
    return result;
}

Example 10 with TServerStatus

use of org.apache.accumulo.core.spi.balancer.data.TServerStatus in project accumulo by apache.

the class ChaoticLoadBalancer method getAssignments.

@Override
public void getAssignments(AssignmentParameters params) {
    long total = params.unassignedTablets().size();
    long avg = (long) Math.ceil(((double) total) / params.currentStatus().size());
    Map<TabletServerId, Long> toAssign = new HashMap<>();
    List<TabletServerId> tServerArray = new ArrayList<>();
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        long numTablets = 0;
        for (TableStatistics ti : e.getValue().getTableMap().values()) {
            numTablets += ti.getTabletCount();
        }
        if (numTablets <= avg) {
            tServerArray.add(e.getKey());
            toAssign.put(e.getKey(), avg - numTablets);
        }
    }
    if (tServerArray.isEmpty()) {
        // No tservers to assign to
        return;
    }
    for (TabletId tabletId : params.unassignedTablets().keySet()) {
        int index = random.nextInt(tServerArray.size());
        TabletServerId dest = tServerArray.get(index);
        params.addAssignment(tabletId, dest);
        long remaining = toAssign.get(dest) - 1;
        if (remaining == 0) {
            tServerArray.remove(index);
            toAssign.remove(dest);
        } else {
            toAssign.put(dest, remaining);
        }
    }
}