Examples with TabletServerId org.apache.accumulo.core.spi.balancer.data.TabletServerId used on opensource projects

Example 11 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class TableLoadBalancerTest method test.

@Test
public void test() {
    BalancerEnvironment environment = createMock(BalancerEnvironment.class);
    ConfigurationCopy cc = new ConfigurationCopy(Map.of(Property.TABLE_LOAD_BALANCER.getKey(), TestSimpleLoadBalancer.class.getName()));
    ConfigurationImpl tableConfig = new ConfigurationImpl(cc);
    Map<String, TableId> tableIdMap = TABLE_ID_MAP.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> TableId.of(e.getValue())));
    expect(environment.getTableIdMap()).andReturn(tableIdMap).anyTimes();
    expect(environment.isTableOnline(anyObject(TableId.class))).andReturn(true).anyTimes();
    expect(environment.getConfiguration(anyObject(TableId.class))).andReturn(tableConfig).anyTimes();
    expect(environment.tableContext(anyObject(TableId.class))).andReturn(null).anyTimes();
    replay(environment);
    String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"), t3Id = TABLE_ID_MAP.get("t3");
    state = new TreeMap<>();
    TabletServerId svr = mkts("10.0.0.1", 1234, "0x01020304");
    state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
    Set<TabletId> migrations = Collections.emptySet();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    TableLoadBalancer tls = new TableLoadBalancer();
    tls.init(environment);
    tls.balance(new BalanceParamsImpl(state, migrations, migrationsOut));
    assertEquals(0, migrationsOut.size());
    state.put(mkts("10.0.0.2", 2345, "0x02030405"), status());
    tls = new TableLoadBalancer();
    tls.init(environment);
    tls.balance(new BalanceParamsImpl(state, migrations, migrationsOut));
    int count = 0;
    Map<TableId, Integer> movedByTable = new HashMap<>();
    movedByTable.put(TableId.of(t1Id), 0);
    movedByTable.put(TableId.of(t2Id), 0);
    movedByTable.put(TableId.of(t3Id), 0);
    for (TabletMigration migration : migrationsOut) {
        if (migration.getOldTabletServer().equals(svr)) {
            count++;
        }
        TableId key = migration.getTablet().getTable();
        movedByTable.put(key, movedByTable.get(key) + 1);
    }
    assertEquals(15, count);
    for (Integer moved : movedByTable.values()) {
        assertEquals(5, moved.intValue());
    }
}

Example 12 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class ChaoticLoadBalancer method balance.

@Override
public long balance(BalanceParameters params) {
    Map<TabletServerId, Long> numTablets = new HashMap<>();
    List<TabletServerId> underCapacityTServer = new ArrayList<>();
    if (!params.currentMigrations().isEmpty()) {
        outstandingMigrationsProblem.setMigrations(params.currentMigrations());
        problemReporter.reportProblem(outstandingMigrationsProblem);
        return 100;
    }
    problemReporter.clearProblemReportTimes();
    boolean moveMetadata = random.nextInt(4) == 0;
    long totalTablets = 0;
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        long tabletCount = 0;
        for (TableStatistics ti : e.getValue().getTableMap().values()) {
            tabletCount += ti.getTabletCount();
        }
        numTablets.put(e.getKey(), tabletCount);
        underCapacityTServer.add(e.getKey());
        totalTablets += tabletCount;
    }
    // totalTablets is fuzzy due to asynchronicity of the stats
    // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
    long avg = (long) Math.ceil(((double) totalTablets) / params.currentStatus().size() * 1.2);
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        for (String tableId : e.getValue().getTableMap().keySet()) {
            TableId id = TableId.of(tableId);
            if (!moveMetadata && MetadataTable.ID.equals(id))
                continue;
            try {
                for (TabletStatistics ts : getOnlineTabletsForTable(e.getKey(), id)) {
                    int index = random.nextInt(underCapacityTServer.size());
                    TabletServerId dest = underCapacityTServer.get(index);
                    if (dest.equals(e.getKey()))
                        continue;
                    params.migrationsOut().add(new TabletMigration(ts.getTabletId(), e.getKey(), dest));
                    if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
                        underCapacityTServer.remove(index);
                    if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
                        underCapacityTServer.add(e.getKey());
                    // option!
                    if (underCapacityTServer.isEmpty())
                        underCapacityTServer.addAll(numTablets.keySet());
                }
            } catch (AccumuloSecurityException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloSecurityException.  This should not happen.  Carrying on anyway.", e1);
            } catch (AccumuloException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloException.  This should not happen.  Carrying on anyway.", e1);
            }
        }
    }
    return 100;
}

Example 13 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class SimpleLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.2", 1234, "b"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.3", 1234, "c"), new FakeTServer());
    List<TabletId> metadataTable = new ArrayList<>();
    String table = "t1";
    metadataTable.add(makeTablet(table, null, null));
    table = "t2";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, null, "a"));
    table = "t3";
    metadataTable.add(makeTablet(table, "a", null));
    metadataTable.add(makeTablet(table, "b", "a"));
    metadataTable.add(makeTablet(table, "c", "b"));
    metadataTable.add(makeTablet(table, "d", "c"));
    metadataTable.add(makeTablet(table, "e", "d"));
    metadataTable.add(makeTablet(table, null, "e"));
    Collections.sort(metadataTable);
    TestSimpleLoadBalancer balancer = new TestSimpleLoadBalancer();
    SortedMap<TabletServerId, TServerStatus> current = new TreeMap<>();
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    assignTablets(metadataTable, servers, current, balancer);
    // Verify that the counts on the tables are correct
    Map<String, Integer> expectedCounts = new HashMap<>();
    expectedCounts.put("t1", 1);
    expectedCounts.put("t2", 1);
    expectedCounts.put("t3", 2);
    checkBalance(metadataTable, servers, expectedCounts);
    // Rebalance once
    for (Entry<TabletServerId, FakeTServer> entry : servers.entrySet()) {
        current.put(entry.getKey(), entry.getValue().getStatus());
    }
    // Nothing should happen, we are balanced
    ArrayList<TabletMigration> out = new ArrayList<>();
    balancer.getMigrations(current, out);
    assertEquals(out.size(), 0);
    // Take down a tabletServer
    TabletServerId first = current.keySet().iterator().next();
    current.remove(first);
    FakeTServer remove = servers.remove(first);
    // reassign offline extents
    assignTablets(remove.tablets, servers, current, balancer);
    checkBalance(metadataTable, servers, null);
}

Example 14 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class HostRegexTableLoadBalancerReconfigurationTest method testConfigurationChanges.

@Test
public void testConfigurationChanges() {
    HashMap<String, TableId> tables = new HashMap<>();
    tables.put(FOO.getTableName(), FOO.getId());
    tables.put(BAR.getTableName(), BAR.getId());
    tables.put(BAZ.getTableName(), BAZ.getId());
    ConfigurationCopy config = new ConfigurationCopy(SiteConfiguration.auto());
    DEFAULT_TABLE_PROPERTIES.forEach(config::set);
    ConfigurationImpl configImpl = new ConfigurationImpl(config);
    BalancerEnvironment environment = createMock(BalancerEnvironment.class);
    expect(environment.getConfiguration()).andReturn(configImpl).anyTimes();
    expect(environment.getTableIdMap()).andReturn(tables).anyTimes();
    expect(environment.getConfiguration(anyObject(TableId.class))).andReturn(configImpl).anyTimes();
    replay(environment);
    init(environment);
    Map<TabletId, TabletServerId> unassigned = new HashMap<>();
    for (List<TabletId> tablets : tableTablets.values()) {
        for (TabletId tablet : tablets) {
            unassigned.put(tablet, null);
        }
    }
    this.getAssignments(new AssignmentParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments));
    assertEquals(15, assignments.size());
    // Ensure unique tservers
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        for (Entry<TabletId, TabletServerId> e2 : assignments.entrySet()) {
            if (e.getKey().equals(e2.getKey())) {
                continue;
            }
            if (e.getValue().equals(e2.getValue())) {
                fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
            }
        }
    }
    // Ensure assignments are correct
    for (Entry<TabletId, TabletServerId> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
        }
    }
    Set<TabletId> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    // Wait to trigger the out of bounds check which will call our version of
    // getOnlineTabletsForTable
    UtilWaitThread.sleep(3000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(0, migrationsOut.size());
    // Change property, simulate call by TableConfWatcher
    config.set(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
    // Wait to trigger the out of bounds check and the repool check
    UtilWaitThread.sleep(10000);
    this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut));
    assertEquals(5, migrationsOut.size());
    for (TabletMigration migration : migrationsOut) {
        assertTrue(migration.getNewTabletServer().getHost().startsWith("192.168.0.1") || migration.getNewTabletServer().getHost().startsWith("192.168.0.2") || migration.getNewTabletServer().getHost().startsWith("192.168.0.3") || migration.getNewTabletServer().getHost().startsWith("192.168.0.4") || migration.getNewTabletServer().getHost().startsWith("192.168.0.5"));
    }
}

Example 15 with TabletServerId

use of org.apache.accumulo.core.spi.balancer.data.TabletServerId in project accumulo by apache.

the class ChaoticLoadBalancerTest method testAssignMigrations.

@Test
public void testAssignMigrations() {
    servers.clear();
    servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.1", 1235, "b"), new FakeTServer());
    servers.put(new TabletServerIdImpl("127.0.0.1", 1236, "c"), new FakeTServer());
    Map<TabletId, TabletServerId> metadataTable = new TreeMap<>();
    String table = "t1";
    metadataTable.put(makeTablet(table, null, null), null);
    table = "t2";
    metadataTable.put(makeTablet(table, "a", null), null);
    metadataTable.put(makeTablet(table, null, "a"), null);
    table = "t3";
    metadataTable.put(makeTablet(table, "a", null), null);
    metadataTable.put(makeTablet(table, "b", "a"), null);
    metadataTable.put(makeTablet(table, "c", "b"), null);
    metadataTable.put(makeTablet(table, "d", "c"), null);
    metadataTable.put(makeTablet(table, "e", "d"), null);
    metadataTable.put(makeTablet(table, null, "e"), null);
    TestChaoticLoadBalancer balancer = new TestChaoticLoadBalancer();
    Map<TabletId, TabletServerId> assignments = new HashMap<>();
    balancer.getAssignments(new AssignmentParamsImpl(getAssignments(servers), metadataTable, assignments));
    assertEquals(assignments.size(), metadataTable.size());
}