Examples with TabletMigration org.apache.accumulo.core.spi.balancer.data.TabletMigration used on opensource projects

Example 1 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class SimpleLoadBalancer method move.

/**
 * Select a tablet based on differences between table loads; if the loads are even, use the
 * busiest table
 */
List<TabletMigration> move(ServerCounts tooMuch, ServerCounts tooLittle, int count, Map<TableId, Map<TabletId, TabletStatistics>> donerTabletStats) {
    if (count == 0) {
        return Collections.emptyList();
    }
    List<TabletMigration> result = new ArrayList<>();
    // Copy counts so we can update them as we propose migrations
    Map<TableId, Integer> tooMuchMap = tabletCountsPerTable(tooMuch.status);
    Map<TableId, Integer> tooLittleMap = tabletCountsPerTable(tooLittle.status);
    for (int i = 0; i < count; i++) {
        TableId table;
        Integer tooLittleCount;
        if (tableToBalance == null) {
            // find a table to migrate
            // look for an uneven table count
            int biggestDifference = 0;
            TableId biggestDifferenceTable = null;
            for (var tableEntry : tooMuchMap.entrySet()) {
                TableId tableID = tableEntry.getKey();
                tooLittleMap.putIfAbsent(tableID, 0);
                int diff = tableEntry.getValue() - tooLittleMap.get(tableID);
                if (diff > biggestDifference) {
                    biggestDifference = diff;
                    biggestDifferenceTable = tableID;
                }
            }
            if (biggestDifference < 2) {
                table = busiest(tooMuch.status.getTableMap());
            } else {
                table = biggestDifferenceTable;
            }
        } else {
            // just balance the given table
            table = tableToBalance;
        }
        Map<TabletId, TabletStatistics> onlineTabletsForTable = donerTabletStats.get(table);
        try {
            if (onlineTabletsForTable == null) {
                onlineTabletsForTable = new HashMap<>();
                List<TabletStatistics> stats = getOnlineTabletsForTable(tooMuch.server, table);
                if (stats == null) {
                    log.warn("Unable to find tablets to move");
                    return result;
                }
                for (TabletStatistics stat : stats) onlineTabletsForTable.put(stat.getTabletId(), stat);
                donerTabletStats.put(table, onlineTabletsForTable);
            }
        } catch (Exception ex) {
            log.error("Unable to select a tablet to move", ex);
            return result;
        }
        TabletId tabletId = selectTablet(onlineTabletsForTable);
        onlineTabletsForTable.remove(tabletId);
        if (tabletId == null)
            return result;
        tooMuchMap.put(table, tooMuchMap.get(table) - 1);
        /*
       * If a table grows from 1 tablet then tooLittleMap.get(table) can return a null, since there
       * is only one tabletserver that holds all of the tablets. Here we check to see if in fact
       * that is the case and if so set the value to 0.
       */
        tooLittleCount = tooLittleMap.get(table);
        if (tooLittleCount == null) {
            tooLittleCount = 0;
        }
        tooLittleMap.put(table, tooLittleCount + 1);
        tooMuch.count--;
        tooLittle.count++;
        result.add(new TabletMigration(tabletId, tooMuch.server, tooLittle.server));
    }
    return result;
}

Example 2 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class GroupBalancer method populateMigrations.

private void populateMigrations(Set<TabletServerId> current, List<TabletMigration> migrationsOut, Moves moves) {
    if (moves.size() == 0) {
        return;
    }
    Function<TabletId, String> partitioner = getPartitioner();
    for (var tablet : getLocationProvider().entrySet()) {
        String group = partitioner.apply(tablet.getKey());
        var loc = tablet.getValue();
        if (loc == null || !current.contains(loc)) {
            migrationsOut.clear();
            return;
        }
        TabletServerId dest = moves.removeMove(loc, group);
        if (dest != null) {
            migrationsOut.add(new TabletMigration(tablet.getKey(), loc, dest));
            if (moves.size() == 0) {
                break;
            }
        }
    }
}

Example 3 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testBalanceWithTooManyOutstandingMigrations.

@Test
public void testBalanceWithTooManyOutstandingMigrations() {
    List<TabletMigration> migrationsOut = new ArrayList<>();
    init(DEFAULT_TABLE_PROPERTIES);
    // lets say we already have migrations ongoing for the FOO and BAR table extends (should be 5 of
    // each of them) for a total of 10
    Set<TabletId> migrations = new HashSet<>();
    migrations.addAll(tableTablets.get(FOO.getTableName()));
    migrations.addAll(tableTablets.get(BAR.getTableName()));
    long wait = this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut));
    assertEquals(20000, wait);
    // no migrations should have occurred as 10 is the maxOutstandingMigrations
    assertEquals(0, migrationsOut.size());
}

Example 4 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testBalance.

@Test
public void testBalance() {
    init(DEFAULT_TABLE_PROPERTIES);
    Set<TabletId> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    long wait = this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut));
    assertEquals(20000, wait);
    // should balance four tablets in one of the tables before reaching max
    assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.getTablet());
    }
    migrationsOut.clear();
    wait = this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut));
    assertEquals(20000, wait);
    // should balance four tablets in one of the other tables before reaching max
    assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.getTablet());
    }
    migrationsOut.clear();
    wait = this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut));
    assertEquals(20000, wait);
    // should balance four tablets in one of the other tables before reaching max
    assertEquals(4, migrationsOut.size());
    // now balance again passing in the new migrations
    for (TabletMigration m : migrationsOut) {
        migrations.add(m.getTablet());
    }
    migrationsOut.clear();
    wait = this.balance(new BalanceParamsImpl(Collections.unmodifiableSortedMap(createCurrent(15)), migrations, migrationsOut));
    assertEquals(20000, wait);
    // no more balancing to do
    assertEquals(0, migrationsOut.size());
}

Example 5 with TabletMigration

use of org.apache.accumulo.core.spi.balancer.data.TabletMigration in project accumulo by apache.

the class TableLoadBalancerTest method test.

@Test
public void test() {
    BalancerEnvironment environment = createMock(BalancerEnvironment.class);
    ConfigurationCopy cc = new ConfigurationCopy(Map.of(Property.TABLE_LOAD_BALANCER.getKey(), TestSimpleLoadBalancer.class.getName()));
    ConfigurationImpl tableConfig = new ConfigurationImpl(cc);
    Map<String, TableId> tableIdMap = TABLE_ID_MAP.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> TableId.of(e.getValue())));
    expect(environment.getTableIdMap()).andReturn(tableIdMap).anyTimes();
    expect(environment.isTableOnline(anyObject(TableId.class))).andReturn(true).anyTimes();
    expect(environment.getConfiguration(anyObject(TableId.class))).andReturn(tableConfig).anyTimes();
    expect(environment.tableContext(anyObject(TableId.class))).andReturn(null).anyTimes();
    replay(environment);
    String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"), t3Id = TABLE_ID_MAP.get("t3");
    state = new TreeMap<>();
    TabletServerId svr = mkts("10.0.0.1", 1234, "0x01020304");
    state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
    Set<TabletId> migrations = Collections.emptySet();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    TableLoadBalancer tls = new TableLoadBalancer();
    tls.init(environment);
    tls.balance(new BalanceParamsImpl(state, migrations, migrationsOut));
    assertEquals(0, migrationsOut.size());
    state.put(mkts("10.0.0.2", 2345, "0x02030405"), status());
    tls = new TableLoadBalancer();
    tls.init(environment);
    tls.balance(new BalanceParamsImpl(state, migrations, migrationsOut));
    int count = 0;
    Map<TableId, Integer> movedByTable = new HashMap<>();
    movedByTable.put(TableId.of(t1Id), 0);
    movedByTable.put(TableId.of(t2Id), 0);
    movedByTable.put(TableId.of(t3Id), 0);
    for (TabletMigration migration : migrationsOut) {
        if (migration.getOldTabletServer().equals(svr)) {
            count++;
        }
        TableId key = migration.getTablet().getTable();
        movedByTable.put(key, movedByTable.get(key) + 1);
    }
    assertEquals(15, count);
    for (Integer moved : movedByTable.values()) {
        assertEquals(5, moved.intValue());
    }
}