Examples with TableStatistics org.apache.accumulo.core.spi.balancer.data.TableStatistics used on opensource projects

Example 1 with TableStatistics

use of org.apache.accumulo.core.spi.balancer.data.TableStatistics in project accumulo by apache.

the class HostRegexTableLoadBalancer method getTableInfo.

/**
 * Get a mutable table info for the specified table and server
 */
private TableStatisticsImpl getTableInfo(SortedMap<TabletServerId, TServerStatusImpl> currentCopy, Multimap<TabletServerId, String> serverTableIdCopied, String tableId, TabletServerId server) {
    TableStatisticsImpl newInfo = null;
    if (currentCopy.containsKey(server)) {
        Map<String, TableStatistics> newTableMap = currentCopy.get(server).getTableMap();
        if (newTableMap != null) {
            newInfo = (TableStatisticsImpl) newTableMap.get(tableId);
            if (newInfo != null) {
                Collection<String> tableIdCopied = serverTableIdCopied.get(server);
                if (tableIdCopied.isEmpty()) {
                    newTableMap = new HashMap<>(newTableMap);
                    currentCopy.get(server).setTableMap(newTableMap);
                }
                if (!tableIdCopied.contains(tableId)) {
                    newInfo = new TableStatisticsImpl(newInfo);
                    newTableMap.put(tableId, newInfo);
                    tableIdCopied.add(tableId);
                }
            }
        }
    }
    return newInfo;
}

Example 2 with TableStatistics

use of org.apache.accumulo.core.spi.balancer.data.TableStatistics in project accumulo by apache.

the class BaseHostRegexTableLoadBalancerTest method createCurrent.

protected SortedMap<TabletServerId, TServerStatus> createCurrent(int numTservers) {
    String base = "192.168.0.";
    TreeMap<TabletServerId, TServerStatus> current = new TreeMap<>();
    for (int i = 1; i <= numTservers; i++) {
        TServerStatusImpl status = new TServerStatusImpl(new org.apache.accumulo.core.master.thrift.TabletServerStatus());
        Map<String, TableStatistics> tableMap = new HashMap<>();
        tableMap.put(FOO.getId().canonical(), new TableStatisticsImpl(new TableInfo()));
        tableMap.put(BAR.getId().canonical(), new TableStatisticsImpl(new TableInfo()));
        tableMap.put(BAZ.getId().canonical(), new TableStatisticsImpl(new TableInfo()));
        status.setTableMap(tableMap);
        current.put(new TabletServerIdImpl(base + i, 9997, Integer.toHexString(1)), status);
    }
    // now put all of the tablets on one server
    for (Map.Entry<String, TabletServerId> entry : initialTableLocation.entrySet()) {
        TServerStatus status = current.get(entry.getValue());
        if (status != null) {
            TableId tableId = environment.getTableIdMap().get(entry.getKey());
            ((TableStatisticsImpl) status.getTableMap().get(tableId.canonical())).setOnlineTabletCount(5);
        }
    }
    return current;
}

Example 3 with TableStatistics

use of org.apache.accumulo.core.spi.balancer.data.TableStatistics in project accumulo by apache.

the class ChaoticLoadBalancer method balance.

@Override
public long balance(BalanceParameters params) {
    Map<TabletServerId, Long> numTablets = new HashMap<>();
    List<TabletServerId> underCapacityTServer = new ArrayList<>();
    if (!params.currentMigrations().isEmpty()) {
        outstandingMigrationsProblem.setMigrations(params.currentMigrations());
        problemReporter.reportProblem(outstandingMigrationsProblem);
        return 100;
    }
    problemReporter.clearProblemReportTimes();
    boolean moveMetadata = random.nextInt(4) == 0;
    long totalTablets = 0;
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        long tabletCount = 0;
        for (TableStatistics ti : e.getValue().getTableMap().values()) {
            tabletCount += ti.getTabletCount();
        }
        numTablets.put(e.getKey(), tabletCount);
        underCapacityTServer.add(e.getKey());
        totalTablets += tabletCount;
    }
    // totalTablets is fuzzy due to asynchronicity of the stats
    // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
    long avg = (long) Math.ceil(((double) totalTablets) / params.currentStatus().size() * 1.2);
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        for (String tableId : e.getValue().getTableMap().keySet()) {
            TableId id = TableId.of(tableId);
            if (!moveMetadata && MetadataTable.ID.equals(id))
                continue;
            try {
                for (TabletStatistics ts : getOnlineTabletsForTable(e.getKey(), id)) {
                    int index = random.nextInt(underCapacityTServer.size());
                    TabletServerId dest = underCapacityTServer.get(index);
                    if (dest.equals(e.getKey()))
                        continue;
                    params.migrationsOut().add(new TabletMigration(ts.getTabletId(), e.getKey(), dest));
                    if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
                        underCapacityTServer.remove(index);
                    if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
                        underCapacityTServer.add(e.getKey());
                    // option!
                    if (underCapacityTServer.isEmpty())
                        underCapacityTServer.addAll(numTablets.keySet());
                }
            } catch (AccumuloSecurityException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloSecurityException.  This should not happen.  Carrying on anyway.", e1);
            } catch (AccumuloException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered AccumuloException.  This should not happen.  Carrying on anyway.", e1);
            }
        }
    }
    return 100;
}

Example 4 with TableStatistics

use of org.apache.accumulo.core.spi.balancer.data.TableStatistics in project accumulo by apache.

the class SimpleLoadBalancer method busiest.

// define what it means for a tablet to be busy
private static TableId busiest(Map<String, TableStatistics> tables) {
    TableId result = null;
    double busiest = Double.NEGATIVE_INFINITY;
    for (Entry<String, TableStatistics> entry : tables.entrySet()) {
        TableStatistics info = entry.getValue();
        double busy = info.getIngestRate() + info.getQueryRate();
        if (busy > busiest) {
            busiest = busy;
            result = TableId.of(entry.getKey());
        }
    }
    return result;
}

Example 5 with TableStatistics

use of org.apache.accumulo.core.spi.balancer.data.TableStatistics in project accumulo by apache.

the class ChaoticLoadBalancer method getAssignments.

@Override
public void getAssignments(AssignmentParameters params) {
    long total = params.unassignedTablets().size();
    long avg = (long) Math.ceil(((double) total) / params.currentStatus().size());
    Map<TabletServerId, Long> toAssign = new HashMap<>();
    List<TabletServerId> tServerArray = new ArrayList<>();
    for (Entry<TabletServerId, TServerStatus> e : params.currentStatus().entrySet()) {
        long numTablets = 0;
        for (TableStatistics ti : e.getValue().getTableMap().values()) {
            numTablets += ti.getTabletCount();
        }
        if (numTablets <= avg) {
            tServerArray.add(e.getKey());
            toAssign.put(e.getKey(), avg - numTablets);
        }
    }
    if (tServerArray.isEmpty()) {
        // No tservers to assign to
        return;
    }
    for (TabletId tabletId : params.unassignedTablets().keySet()) {
        int index = random.nextInt(tServerArray.size());
        TabletServerId dest = tServerArray.get(index);
        params.addAssignment(tabletId, dest);
        long remaining = toAssign.get(dest) - 1;
        if (remaining == 0) {
            tServerArray.remove(index);
            toAssign.remove(dest);
        } else {
            toAssign.put(dest, remaining);
        }
    }
}