Examples with TabletMigration org.apache.accumulo.core.spi.balancer.data.TabletMigration used on opensource projects

Example 6 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class ChaoticLoadBalancer method balance.

@Override
public long balance(BalanceParameters params) {
    Map<TabletServerId, Long> numTablets = new HashMap<>();
    List<TabletServerId> underCapacityTServer = new ArrayList<>();
    if (!params.currentMigrations().isEmpty()) {
        outstandingMigrationsProblem.setMigrations(params.currentMigrations());
        problemReporter.reportProblem(outstandingMigrationsProblem);
        return 100;
    }
    problemReporter.clearProblemReportTimes();
    boolean moveMetadata = random.nextInt(4) == 0;
    long totalTablets = 0;
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        long tabletCount = 0;
        for (TableStatistics ti : e.getValue().getTableMap().values()) {
            tabletCount += ti.getTabletCount();
        }
        numTablets.put(e.getKey(), tabletCount);
        underCapacityTServer.add(e.getKey());
        totalTablets += tabletCount;
    }
    // totalTablets is fuzzy due to asynchronicity of the stats
    // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
    long avg = (long) Math.ceil(((double) totalTablets) / params.currentStatus().size() * 1.2);
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        for (String tableId : e.getValue().getTableMap().keySet()) {
            TableId id = TableId.of(tableId);
            if (!moveMetadata && MetadataTable.ID.equals(id))
                continue;
            try {
                for (TabletStatistics ts : getOnlineTabletsForTable(e.getKey(), id)) {
                    int index = random.nextInt(underCapacityTServer.size());
                    TabletServerId dest = underCapacityTServer.get(index);
                    if (dest.equals(e.getKey()))
                        continue;
                    params.migrationsOut().add(new TabletMigration(ts.getTabletId(), e.getKey(), dest));
                    if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
                        underCapacityTServer.remove(index);
                    if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
                        underCapacityTServer.add(e.getKey());
                    // option!
                    if (underCapacityTServer.isEmpty())
                        underCapacityTServer.addAll(numTablets.keySet());
                }
            } catch (AccumuloSecurityException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloSecurityException.  This should not happen.  Carrying on anyway.", e1);
            } catch (AccumuloException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloException.  This should not happen.  Carrying on anyway.", e1);
            }
        }
    }
    return 100;
}

Example 7 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class SimpleLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.2", 1234, "b"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.3", 1234, "c"), new FakeTServer());
    List<TabletId> metadataTable = new ArrayList<>();
    String table = "t1";
    metadataTable.add(makeTablet(table, null, null));
    table = "t2";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, null, "a"));
    table = "t3";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, "b", "a"));
    metadataTable.add(makeTablet(table, "c", "b"));
    metadataTable.add(makeTablet(table, "d", "c"));
    metadataTable.add(makeTablet(table, "e", "d"));
    metadataTable.add(makeTablet(table, null, "e"));
    Collections.sort(metadataTable);
    TestSimpleLoadBalancer balancer = new TestSimpleLoadBalancer();
    SortedMap<TabletServerId, TServerStatus> current = new TreeMap<>();
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    assignTablets(metadataTable, servers, current, balancer);
    // Verify that the counts on the tables are correct
    Map<String, Integer> expectedCounts = new HashMap<>();
    expectedCounts.put("t1", 1);
    expectedCounts.put("t2", 1);
    expectedCounts.put("t3", 2);
    checkBalance(metadataTable, servers, expectedCounts);
    // Rebalance once
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    // Nothing should happen, we are balanced
    ArrayList<TabletMigration> out = new ArrayList<>();
    balancer.getMigrations(current, out);
    assertEquals(out.size(), 0);
    // Take down a tabletServer
    TabletServerId first = current.keySet().iterator().next();
    current.remove(first);
    FakeTServer remove = servers.remove(first);
    // reassign offline extents
    assignTablets(remove.tablets, servers, current, balancer);
    checkBalance(metadataTable, servers, null);
}

Example 8 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class HostRegexTableLoadBalancerReconfigurationTest method testConfigurationChanges.

@Test
public void testConfigurationChanges() {
    HashMap<String, TableId> tables = new HashMap<>();
    tables.put(FOO.getTableName(), FOO.getId());
    tables.put(BAR.getTableName(), BAR.getId());
    tables.put(BAZ.getTableName(), BAZ.getId());
    ConfigurationCopy config = new ConfigurationCopy(SiteConfiguration.auto());
    DEFAULT_TABLE_PROPERTIES.forEach(config::set);
    ConfigurationImpl configImpl = new ConfigurationImpl(config);
    BalancerEnvironment environment = createMock(BalancerEnvironment.class);
    expect(environment.getConfiguration()).andReturn(configImpl).anyTimes();
    expect(environment.getTableIdMap()).andReturn(tables).anyTimes();
    expect(environment.getConfiguration(anyObject(TableId.class))).andReturn(configImpl).anyTimes();
    replay(environment);
    init(environment);
    Map<TabletId, TabletServerId> unassigned = new HashMap<>();
    for (List<TabletId> tablets : tableTablets.values()) {
        for (TabletId tablet : tablets) {
            unassigned.put(tablet, null);
        }
    }
    this.getAssignments(new AssignmentParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments));
    assertEquals(15, assignments.size());
    // Ensure unique tservers
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        for (Entry<TabletId, TabletServerId> e2 : assignments.entrySet()) {
            if (e.getKey().equals(e2.getKey())) {
                continue;
            }
            if (e.getValue().equals(e2.getValue())) {
                fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
            }
        }
    }
    // Ensure assignments are correct
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
        }
    }
    Set<TabletId> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    // Wait to trigger the out of bounds check which will call our version of
    // getOnlineTabletsForTable
    UtilWaitThread.sleep(3000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(0, migrationsOut.size());
    // Change property, simulate call by TableConfWatcher
    config.set(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
    // Wait to trigger the out of bounds check and the repool check
    UtilWaitThread.sleep(10000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(5, migrationsOut.size());
    for (TabletMigration migration : migrationsOut) {
        assertTrue(migration.getNewTabletServer().getHost().startsWith("192.168.0.1") || migration.getNewTabletServer().getHost().startsWith("192.168.0.2") || migration.getNewTabletServer().getHost().startsWith("192.168.0.3") || migration.getNewTabletServer().getHost().startsWith("192.168.0.4") || migration.getNewTabletServer().getHost().startsWith("192.168.0.5"));
    }
}

Example 9 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class ChaoticLoadBalancerTest method testUnevenAssignment.

@Test
public void testUnevenAssignment() {
    servers.clear();
    for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
        String cString = Character.toString(c);
        TabletServerId tsi = new TabletServerIdImpl("127.0.0.1", c, cString);
        FakeTServer fakeTServer = new FakeTServer();
        servers.put(tsi, fakeTServer);
        fakeTServer.tablets.add(makeTablet(cString, null, null));
    }
    // Put more tablets on one server, but not more than the number of servers
    Entry<TabletServerId, FakeTServer> first = servers.entrySet().iterator().next();
    first.getValue().tablets.add(makeTablet("newTable", "a", null));
    first.getValue().tablets.add(makeTablet("newTable", "b", "a"));
    first.getValue().tablets.add(makeTablet("newTable", "c", "b"));
    first.getValue().tablets.add(makeTablet("newTable", "d", "c"));
    first.getValue().tablets.add(makeTablet("newTable", "e", "d"));
    first.getValue().tablets.add(makeTablet("newTable", "f", "e"));
    first.getValue().tablets.add(makeTablet("newTable", "g", "f"));
    first.getValue().tablets.add(makeTablet("newTable", "h", "g"));
    first.getValue().tablets.add(makeTablet("newTable", "i", null));
    TestChaoticLoadBalancer balancer = new TestChaoticLoadBalancer();
    Set<TabletId> migrations = Collections.emptySet();
    // Just want to make sure it gets some migrations, randomness prevents guarantee of a defined
    // amount, or even expected amount
    List<TabletMigration> migrationsOut = new ArrayList<>();
    while (!migrationsOut.isEmpty()) {
        balancer.balance(new BalanceParamsImpl(getAssignments(servers), migrations, migrationsOut));
    }
}

Example 10 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class BalanceParamsImpl method addMigration.

public void addMigration(KeyExtent extent, TServerInstance oldServer, TServerInstance newServer) {
    TabletId id = new TabletIdImpl(extent);
    TabletServerId oldTsid = new TabletServerIdImpl(oldServer);
    TabletServerId newTsid = new TabletServerIdImpl(newServer);
    migrationsOut.add(new TabletMigration(id, oldTsid, newTsid));
}