Examples with TableInfo org.apache.accumulo.core.master.thrift.TableInfo used on opensource projects

Example 11 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class HostRegexTableLoadBalancer method getTableInfo.

/**
 * Get a mutable table info for the specified table and server
 */
private TableInfo getTableInfo(SortedMap<TServerInstance, TabletServerStatus> currentCopy, Multimap<TServerInstance, String> serverTableIdCopied, String tableId, TServerInstance server) {
    TableInfo newInfo = null;
    if (currentCopy.containsKey(server)) {
        Map<String, TableInfo> newTableMap = currentCopy.get(server).getTableMap();
        if (newTableMap != null) {
            newInfo = newTableMap.get(tableId);
            if (newInfo != null) {
                Collection<String> tableIdCopied = serverTableIdCopied.get(server);
                if (tableIdCopied.isEmpty()) {
                    newTableMap = new HashMap<>(newTableMap);
                    currentCopy.get(server).setTableMap(newTableMap);
                }
                if (!tableIdCopied.contains(tableId)) {
                    newInfo = new TableInfo(newInfo);
                    newTableMap.put(tableId, newInfo);
                    tableIdCopied.add(tableId);
                }
            }
        }
    }
    return newInfo;
}

Example 12 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class HostRegexTableLoadBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    long minBalanceTime = 20 * 1000;
    // Iterate over the tables and balance each of them
    TableOperations t = getTableOperations();
    if (t == null)
        return minBalanceTime;
    Map<String, String> tableIdMap = t.tableIdMap();
    long now = System.currentTimeMillis();
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> currentGrouped = splitCurrentByRegex(current);
    if ((now - this.lastOOBCheck) > this.oobCheckMillis) {
        try {
            // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
            for (String table : t.list()) {
                LOG.debug("Checking for out of bounds tablets for table {}", table);
                String tablePoolName = getPoolNameForTable(table);
                for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
                    // pool names are the same as table names, except in the DEFAULT case.
                    // If this table is assigned to a pool for this host, then move on.
                    List<String> hostPools = getPoolNamesForHost(e.getKey().host());
                    if (hostPools.contains(tablePoolName)) {
                        continue;
                    }
                    String tid = tableIdMap.get(table);
                    if (null == tid) {
                        LOG.warn("Unable to check for out of bounds tablets for table {}, it may have been deleted or renamed.", table);
                        continue;
                    }
                    try {
                        List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), Table.ID.of(tid));
                        if (null == outOfBoundsTablets) {
                            continue;
                        }
                        Random random = new Random();
                        for (TabletStats ts : outOfBoundsTablets) {
                            KeyExtent ke = new KeyExtent(ts.getExtent());
                            if (migrations.contains(ke)) {
                                LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
                                continue;
                            }
                            String poolName = getPoolNameForTable(table);
                            SortedMap<TServerInstance, TabletServerStatus> currentView = currentGrouped.get(poolName);
                            if (null != currentView) {
                                int skip = random.nextInt(currentView.size());
                                Iterator<TServerInstance> iter = currentView.keySet().iterator();
                                for (int i = 0; i < skip; i++) {
                                    iter.next();
                                }
                                TServerInstance nextTS = iter.next();
                                LOG.info("Tablet {} is currently outside the bounds of the regex, migrating from {} to {}", ke, e.getKey(), nextTS);
                                migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
                                if (migrationsOut.size() >= this.maxTServerMigrations) {
                                    break;
                                }
                            } else {
                                LOG.warn("No tablet servers online for pool {}, unable to migrate out of bounds tablets", poolName);
                            }
                        }
                    } catch (TException e1) {
                        LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e);
                    }
                }
            }
        } finally {
            // this could have taken a while...get a new time
            this.lastOOBCheck = System.currentTimeMillis();
        }
    }
    if (migrationsOut.size() > 0) {
        LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
        LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
        return minBalanceTime;
    }
    if (migrations != null && migrations.size() > 0) {
        if (migrations.size() >= maxOutstandingMigrations) {
            LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
            if (LOG.isTraceEnabled()) {
                LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
            }
            return minBalanceTime;
        }
        LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
        }
        migrationsFromLastPass.keySet().retainAll(migrations);
        SortedMap<TServerInstance, TabletServerStatus> currentCopy = new TreeMap<>(current);
        Multimap<TServerInstance, String> serverTableIdCopied = HashMultimap.create();
        for (TabletMigration migration : migrationsFromLastPass.values()) {
            TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(), migration.oldServer);
            if (fromInfo != null) {
                fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1);
            }
            TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.getTableId().toString(), migration.newServer);
            if (toInfo != null) {
                toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1);
            }
        }
        migrations = EMPTY_MIGRATIONS;
    } else {
        migrationsFromLastPass.clear();
    }
    for (String s : tableIdMap.values()) {
        Table.ID tableId = Table.ID.of(s);
        String tableName = tableIdToTableName.get(tableId);
        String regexTableName = getPoolNameForTable(tableName);
        SortedMap<TServerInstance, TabletServerStatus> currentView = currentGrouped.get(regexTableName);
        if (null == currentView) {
            LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
            continue;
        }
        ArrayList<TabletMigration> newMigrations = new ArrayList<>();
        getBalancerForTable(tableId).balance(currentView, migrations, newMigrations);
        if (newMigrations.isEmpty()) {
            tableToTimeSinceNoMigrations.remove(s);
        } else if (tableToTimeSinceNoMigrations.containsKey(s)) {
            if ((now - tableToTimeSinceNoMigrations.get(s)) > ONE_HOUR) {
                LOG.warn("We have been consistently producing migrations for {}: {}", tableName, Iterables.limit(newMigrations, 10));
            }
        } else {
            tableToTimeSinceNoMigrations.put(s, now);
        }
        migrationsOut.addAll(newMigrations);
        if (migrationsOut.size() >= this.maxTServerMigrations) {
            break;
        }
    }
    for (TabletMigration migration : migrationsOut) {
        migrationsFromLastPass.put(migration.tablet, migration);
    }
    LOG.info("Migrating tablets for balance: {}", migrationsOut);
    return minBalanceTime;
}

Example 13 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class TableLoadBalancerTest method status.

private static TabletServerStatus status(Object... config) {
    TabletServerStatus result = new TabletServerStatus();
    result.tableMap = new HashMap<>();
    String tablename = null;
    for (Object c : config) {
        if (c instanceof String) {
            tablename = (String) c;
        } else {
            TableInfo info = new TableInfo();
            int count = (Integer) c;
            info.onlineTablets = count;
            info.tablets = count;
            result.tableMap.put(tablename, info);
        }
    }
    return result;
}

Example 14 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class DynamicThreadPoolsIT method test.

@Test
public void test() throws Exception {
    final String[] tables = getUniqueNames(15);
    String firstTable = tables[0];
    Connector c = getConnector();
    c.instanceOperations().setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "5");
    TestIngest.Opts opts = new TestIngest.Opts();
    opts.rows = 500 * 1000;
    opts.createTable = true;
    opts.setTableName(firstTable);
    ClientConfiguration clientConf = cluster.getClientConfig();
    if (clientConf.hasSasl()) {
        opts.updateKerberosCredentials(clientConf);
    } else {
        opts.setPrincipal(getAdminPrincipal());
    }
    TestIngest.ingest(c, opts, new BatchWriterOpts());
    c.tableOperations().flush(firstTable, null, null, true);
    for (int i = 1; i < tables.length; i++) c.tableOperations().clone(firstTable, tables[i], true, null, null);
    // time between checks of the thread pool sizes
    sleepUninterruptibly(11, TimeUnit.SECONDS);
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    for (int i = 1; i < tables.length; i++) c.tableOperations().compact(tables[i], null, null, true, false);
    for (int i = 0; i < 30; i++) {
        int count = 0;
        MasterClientService.Iface client = null;
        MasterMonitorInfo stats = null;
        while (true) {
            try {
                client = MasterClient.getConnectionWithRetry(new ClientContext(c.getInstance(), creds, clientConf));
                stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(c.getInstance()));
                break;
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } finally {
                if (client != null)
                    MasterClient.close(client);
            }
        }
        for (TabletServerStatus server : stats.tServerInfo) {
            for (TableInfo table : server.tableMap.values()) {
                count += table.majors.running;
            }
        }
        System.out.println("count " + count);
        if (count > 3)
            return;
        sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    }
    fail("Could not observe higher number of threads after changing the config");
}

Example 15 with TableInfo

use of org.apache.accumulo.core.master.thrift.TableInfo in project accumulo by apache.

the class TableInfoUtil method summarizeTableStats.

public static TableInfo summarizeTableStats(TabletServerStatus status) {
    TableInfo summary = new TableInfo();
    summary.majors = new Compacting();
    summary.minors = new Compacting();
    summary.scans = new Compacting();
    for (TableInfo rates : status.tableMap.values()) {
        TableInfoUtil.add(summary, rates);
    }
    return summary;
}