Examples with TabletMigration org.apache.accumulo.server.master.state.TabletMigration used on opensource projects

Example 1 with TabletMigration

use of org.apache.accumulo.server.master.state.TabletMigration in project accumulo by apache.

the class DefaultLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), "a"), new FakeTServer());
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.2", 1234), "b"), new FakeTServer());
    servers.put(new TServerInstance(HostAndPort.fromParts("127.0.0.3", 1234), "c"), new FakeTServer());
    List<KeyExtent> metadataTable = new ArrayList<>();
    String table = "t1";
    metadataTable.add(makeExtent(table, null, null));
    table = "t2";
    metadataTable.add(makeExtent(table, "a", null));
    metadataTable.add(makeExtent(table, null, "a"));
    table = "t3";
    metadataTable.add(makeExtent(table, "a", null));
    metadataTable.add(makeExtent(table, "b", "a"));
    metadataTable.add(makeExtent(table, "c", "b"));
    metadataTable.add(makeExtent(table, "d", "c"));
    metadataTable.add(makeExtent(table, "e", "d"));
    metadataTable.add(makeExtent(table, null, "e"));
    Collections.sort(metadataTable);
    TestDefaultLoadBalancer balancer = new TestDefaultLoadBalancer();
    SortedMap<TServerInstance, TabletServerStatus> current = new TreeMap<>();
    for (Entry<TServerInstance, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus(entry.getKey()));
    }
    assignTablets(metadataTable, servers, current, balancer);
    // Verify that the counts on the tables are correct
    Map<String, Integer> expectedCounts = new HashMap<>();
    expectedCounts.put("t1", 1);
    expectedCounts.put("t2", 1);
    expectedCounts.put("t3", 2);
    checkBalance(metadataTable, servers, expectedCounts);
    // Rebalance once
    for (Entry<TServerInstance, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus(entry.getKey()));
    }
    // Nothing should happen, we are balanced
    ArrayList<TabletMigration> out = new ArrayList<>();
    balancer.getMigrations(current, out);
    assertEquals(out.size(), 0);
    // Take down a tabletServer
    TServerInstance first = current.keySet().iterator().next();
    current.remove(first);
    FakeTServer remove = servers.remove(first);
    // reassign offline extents
    assignTablets(remove.extents, servers, current, balancer);
    checkBalance(metadataTable, servers, null);
}

Example 2 with TabletMigration

use of org.apache.accumulo.server.master.state.TabletMigration in project accumulo by apache.

the class HostRegexTableLoadBalancerReconfigurationTest method testConfigurationChanges.

@Test
public void testConfigurationChanges() {
    init(new AccumuloServerContext(instance, factory));
    Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
    for (List<KeyExtent> extents : tableExtents.values()) {
        for (KeyExtent ke : extents) {
            unassigned.put(ke, null);
        }
    }
    this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
    Assert.assertEquals(15, assignments.size());
    // Ensure unique tservers
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        for (Entry<KeyExtent, TServerInstance> e2 : assignments.entrySet()) {
            if (e.getKey().equals(e2.getKey())) {
                continue;
            }
            if (e.getValue().equals(e2.getValue())) {
                Assert.fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
            }
        }
    }
    // Ensure assignments are correct
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            Assert.fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().host());
        }
    }
    Set<KeyExtent> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
    UtilWaitThread.sleep(3000);
    this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
    Assert.assertEquals(0, migrationsOut.size());
    // Change property, simulate call by TableConfWatcher
    DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
    this.propertiesChanged();
    // Wait to trigger the out of bounds check and the repool check
    UtilWaitThread.sleep(10000);
    this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
    Assert.assertEquals(5, migrationsOut.size());
    for (TabletMigration migration : migrationsOut) {
        Assert.assertTrue(migration.newServer.host().startsWith("192.168.0.1") || migration.newServer.host().startsWith("192.168.0.2") || migration.newServer.host().startsWith("192.168.0.3") || migration.newServer.host().startsWith("192.168.0.4") || migration.newServer.host().startsWith("192.168.0.5"));
    }
}

Example 3 with TabletMigration

use of org.apache.accumulo.server.master.state.TabletMigration in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testBalance.

@Test
public void testBalance() {
    init(new AccumuloServerContext(instance, factory));
    Set<KeyExtent> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    long wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut);
    Assert.assertEquals(20000, wait);
    // should balance four tablets in one of the tables before reaching max
    Assert.assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.tablet);
    }
    migrationsOut.clear();
    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut);
    Assert.assertEquals(20000, wait);
    // should balance four tablets in one of the other tables before reaching max
    Assert.assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.tablet);
    }
    migrationsOut.clear();
    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut);
    Assert.assertEquals(20000, wait);
    // should balance four tablets in one of the other tables before reaching max
    Assert.assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.tablet);
    }
    migrationsOut.clear();
    wait = this.balance(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut);
    Assert.assertEquals(20000, wait);
    // no more balancing to do
    Assert.assertEquals(0, migrationsOut.size());
}

Example 4 with TabletMigration

use of org.apache.accumulo.server.master.state.TabletMigration in project accumulo by apache.

the class ChaoticLoadBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    Map<TServerInstance, Long> numTablets = new HashMap<>();
    List<TServerInstance> underCapacityTServer = new ArrayList<>();
    if (!migrations.isEmpty()) {
        outstandingMigrations.migrations = migrations;
        constraintNotMet(outstandingMigrations);
        return 100;
    }
    resetBalancerErrors();
    boolean moveMetadata = r.nextInt(4) == 0;
    long totalTablets = 0;
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        long tabletCount = 0;
        for (TableInfo ti : e.getValue().getTableMap().values()) {
            tabletCount += ti.tablets;
        }
        numTablets.put(e.getKey(), tabletCount);
        underCapacityTServer.add(e.getKey());
        totalTablets += tabletCount;
    }
    // totalTablets is fuzzy due to asynchronicity of the stats
    // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
    long avg = (long) Math.ceil(((double) totalTablets) / current.size() * 1.2);
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        for (String tableId : e.getValue().getTableMap().keySet()) {
            Table.ID id = Table.ID.of(tableId);
            if (!moveMetadata && MetadataTable.ID.equals(id))
                continue;
            try {
                for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), id)) {
                    KeyExtent ke = new KeyExtent(ts.extent);
                    int index = r.nextInt(underCapacityTServer.size());
                    TServerInstance dest = underCapacityTServer.get(index);
                    if (dest.equals(e.getKey()))
                        continue;
                    migrationsOut.add(new TabletMigration(ke, e.getKey(), dest));
                    if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
                        underCapacityTServer.remove(index);
                    if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
                        underCapacityTServer.add(e.getKey());
                    // We can get some craziness with only 1 tserver, so lets make sure there's always an option!
                    if (underCapacityTServer.isEmpty())
                        underCapacityTServer.addAll(numTablets.keySet());
                }
            } catch (ThriftSecurityException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered ThriftSecurityException.  This should not happen.  Carrying on anyway.", e1);
            } catch (TException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered TException.  This should not happen.  Carrying on anyway.", e1);
            }
        }
    }
    return 100;
}

Example 5 with TabletMigration

use of org.apache.accumulo.server.master.state.TabletMigration in project accumulo by apache.

the class DefaultLoadBalancer method move.

/**
 * Select a tablet based on differences between table loads; if the loads are even, use the busiest table
 */
List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count, Map<Table.ID, Map<KeyExtent, TabletStats>> donerTabletStats) {
    List<TabletMigration> result = new ArrayList<>();
    if (count == 0)
        return result;
    // Copy counts so we can update them as we propose migrations
    Map<Table.ID, Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
    Map<Table.ID, Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
    for (int i = 0; i < count; i++) {
        Table.ID table;
        Integer tooLittleCount;
        if (tableToBalance == null) {
            // find a table to migrate
            // look for an uneven table count
            int biggestDifference = 0;
            Table.ID biggestDifferenceTable = null;
            for (Entry<Table.ID, Integer> tableEntry : tooMuchMap.entrySet()) {
                Table.ID tableID = tableEntry.getKey();
                if (tooLittleMap.get(tableID) == null)
                    tooLittleMap.put(tableID, 0);
                int diff = tableEntry.getValue() - tooLittleMap.get(tableID);
                if (diff > biggestDifference) {
                    biggestDifference = diff;
                    biggestDifferenceTable = tableID;
                }
            }
            if (biggestDifference < 2) {
                table = busiest(tooMuch.status.tableMap);
            } else {
                table = biggestDifferenceTable;
            }
        } else {
            // just balance the given table
            table = tableToBalance;
        }
        Map<KeyExtent, TabletStats> onlineTabletsForTable = donerTabletStats.get(table);
        try {
            if (onlineTabletsForTable == null) {
                onlineTabletsForTable = new HashMap<>();
                List<TabletStats> stats = getOnlineTabletsForTable(tooMuch.server, table);
                if (null == stats) {
                    log.warn("Unable to find tablets to move");
                    return result;
                }
                for (TabletStats stat : stats) onlineTabletsForTable.put(new KeyExtent(stat.extent), stat);
                donerTabletStats.put(table, onlineTabletsForTable);
            }
        } catch (Exception ex) {
            log.error("Unable to select a tablet to move", ex);
            return result;
        }
        KeyExtent extent = selectTablet(tooMuch.server, onlineTabletsForTable);
        onlineTabletsForTable.remove(extent);
        if (extent == null)
            return result;
        tooMuchMap.put(table, tooMuchMap.get(table) - 1);
        /**
         * If a table grows from 1 tablet then tooLittleMap.get(table) can return a null, since there is only one tabletserver that holds all of the tablets. Here
         * we check to see if in fact that is the case and if so set the value to 0.
         */
        tooLittleCount = tooLittleMap.get(table);
        if (tooLittleCount == null) {
            tooLittleCount = 0;
        }
        tooLittleMap.put(table, tooLittleCount + 1);
        tooMuch.count--;
        tooLittle.count++;
        result.add(new TabletMigration(extent, tooMuch.server, tooLittle.server));
    }
    return result;
}