Examples with TServerInstance org.apache.accumulo.core.metadata.TServerInstance used on opensource projects

Example 51 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingOverlappingPools.

@Test
public void testSplitCurrentByRegexUsingOverlappingPools() {
    ServerContext context = createMockContext();
    replay(context);
    initFactory(new TestServerConfigurationFactory(context) {

        @Override
        public synchronized AccumuloConfiguration getSystemConfiguration() {
            HashMap<String, String> props = new HashMap<>(DEFAULT_TABLE_PROPERTIES);
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r.*");
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*|r02.*");
            return new ConfigurationCopy(props);
        }
    });
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
    // Groups foo, bar, and the default pool which contains all known hosts
    assertEquals(3, groups.size());
    assertTrue(groups.containsKey(FOO.getTableName()));
    assertTrue(groups.containsKey(DEFAULT_POOL));
    for (String pool : new String[] { FOO.getTableName(), DEFAULT_POOL }) {
        SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(pool);
        assertEquals(15, fooHosts.size());
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
        assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
    }
    assertTrue(groups.containsKey(BAR.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
    assertEquals(10, barHosts.size());
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
}

Example 52 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingIP.

@Test
public void testSplitCurrentByRegexUsingIP() {
    ServerContext context = createMockContext();
    replay(context);
    initFactory(new TestServerConfigurationFactory(context) {

        @Override
        public synchronized AccumuloConfiguration getSystemConfiguration() {
            HashMap<String, String> props = new HashMap<>();
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "30s");
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS_KEY, "true");
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "192\\.168\\.0\\.[1-5]");
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "192\\.168\\.0\\.[6-9]|192\\.168\\.0\\.10");
            return new ConfigurationCopy(props);
        }
    });
    assertTrue(isIpBasedRegex());
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
    assertEquals(3, groups.size());
    assertTrue(groups.containsKey(FOO.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
    assertEquals(5, fooHosts.size());
    assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
    assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
    assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
    assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
    assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
    assertTrue(groups.containsKey(BAR.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
    assertEquals(5, barHosts.size());
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
    assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
    assertTrue(groups.containsKey(DEFAULT_POOL));
    SortedMap<TServerInstance, TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
    assertEquals(5, defHosts.size());
    assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
    assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
    assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
    assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
    assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
}

Example 53 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testUnassignedWithNoTServers.

@Test
public void testUnassignedWithNoTServers() {
    init();
    Map<KeyExtent, TServerInstance> assignments = new HashMap<>();
    Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
    for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
        unassigned.put(ke, null);
    }
    SortedMap<TServerInstance, TabletServerStatus> current = createCurrent(15);
    // Remove the BAR tablet servers from current
    List<TServerInstance> removals = new ArrayList<>();
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        if (e.getKey().getHost().equals("192.168.0.6") || e.getKey().getHost().equals("192.168.0.7") || e.getKey().getHost().equals("192.168.0.8") || e.getKey().getHost().equals("192.168.0.9") || e.getKey().getHost().equals("192.168.0.10")) {
            removals.add(e.getKey());
        }
    }
    for (TServerInstance r : removals) {
        current.remove(r);
    }
    this.getAssignments(Collections.unmodifiableSortedMap(current), Collections.unmodifiableMap(unassigned), assignments);
    assertEquals(unassigned.size(), assignments.size());
    // Ensure tablets are assigned in default pool
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        if (tabletInBounds(e.getKey(), e.getValue())) {
            fail("tablet unexpectedly in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
        }
    }
}

Example 54 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class QueueSummariesTest method update.

private void update(QueueSummaries queueSum, String tserver, String... data) {
    TServerInstance tsi = ntsi(tserver);
    List<TCompactionQueueSummary> summaries = new ArrayList<>();
    for (int i = 0; i < data.length; i += 2) {
        summaries.add(new TCompactionQueueSummary(data[i], Short.parseShort(data[i + 1])));
    }
    queueSum.update(tsi, summaries);
}

Example 55 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class HostRegexTableLoadBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    long minBalanceTime = 20_000;
    // Iterate over the tables and balance each of them
    TableOperations t = getTableOperations();
    if (t == null) {
        return minBalanceTime;
    }
    Map<String, String> tableIdMap = t.tableIdMap();
    Map<TableId, String> tableIdToTableName = createdTableNameMap(tableIdMap);
    tableIdToTableName.keySet().forEach(this::checkTableConfig);
    long now = System.currentTimeMillis();
    HrtlbConf myConf = hrtlbConf.derive();
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> currentGrouped = splitCurrentByRegex(current);
    if ((now - this.lastOOBCheck) > myConf.oobCheckMillis) {
        try {
            // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
            for (String table : tableIdMap.keySet()) {
                LOG.debug("Checking for out of bounds tablets for table {}", table);
                String tablePoolName = getPoolNameForTable(table);
                for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
                    // pool names are the same as table names, except in the DEFAULT case.
                    // If this table is assigned to a pool for this host, then move on.
                    List<String> hostPools = getPoolNamesForHost(e.getKey().getHost());
                    if (hostPools.contains(tablePoolName)) {
                        continue;
                    }
                    String tid = tableIdMap.get(table);
                    if (tid == null) {
                        LOG.warn("Unable to check for out of bounds tablets for table {}," + " it may have been deleted or renamed.", table);
                        continue;
                    }
                    try {
                        List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), TableId.of(tid));
                        if (outOfBoundsTablets == null) {
                            continue;
                        }
                        for (TabletStats ts : outOfBoundsTablets) {
                            KeyExtent ke = KeyExtent.fromThrift(ts.getExtent());
                            if (migrations.contains(ke)) {
                                LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
                                continue;
                            }
                            String poolName = getPoolNameForTable(table);
                            SortedMap<TServerInstance, TabletServerStatus> currentView = currentGrouped.get(poolName);
                            if (currentView != null) {
                                int skip = random.nextInt(currentView.size());
                                Iterator<TServerInstance> iter = currentView.keySet().iterator();
                                for (int i = 0; i < skip; i++) {
                                    iter.next();
                                }
                                TServerInstance nextTS = iter.next();
                                LOG.info("Tablet {} is currently outside the bounds of the" + " regex, migrating from {} to {}", ke, e.getKey(), nextTS);
                                migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
                                if (migrationsOut.size() >= myConf.maxTServerMigrations) {
                                    break;
                                }
                            } else {
                                LOG.warn("No tablet servers online for pool {}, unable to" + " migrate out of bounds tablets", poolName);
                            }
                        }
                    } catch (TException e1) {
                        LOG.error("Error in OOB check getting tablets for table {} from server {} {}", tid, e.getKey().getHost(), e);
                    }
                }
            }
        } finally {
            // this could have taken a while...get a new time
            this.lastOOBCheck = System.currentTimeMillis();
        }
    }
    if (!migrationsOut.isEmpty()) {
        LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
        LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
        return minBalanceTime;
    }
    if (migrations != null && !migrations.isEmpty()) {
        if (migrations.size() >= myConf.maxOutstandingMigrations) {
            LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
            if (LOG.isTraceEnabled()) {
                LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
            }
            return minBalanceTime;
        }
        LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
        if (LOG.isTraceEnabled()) {
            LOG.trace("Sample up to 10 outstanding migrations: {}", Iterables.limit(migrations, 10));
        }
        migrationsFromLastPass.keySet().retainAll(migrations);
        SortedMap<TServerInstance, TabletServerStatus> currentCopy = new TreeMap<>(current);
        Multimap<TServerInstance, String> serverTableIdCopied = HashMultimap.create();
        for (TabletMigration migration : migrationsFromLastPass.values()) {
            TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.tableId().toString(), migration.oldServer);
            if (fromInfo != null) {
                fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1);
            }
            TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied, migration.tablet.tableId().toString(), migration.newServer);
            if (toInfo != null) {
                toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1);
            }
        }
        migrations = EMPTY_MIGRATIONS;
    } else {
        migrationsFromLastPass.clear();
    }
    for (String s : tableIdMap.values()) {
        TableId tableId = TableId.of(s);
        String tableName = tableIdToTableName.get(tableId);
        String regexTableName = getPoolNameForTable(tableName);
        SortedMap<TServerInstance, TabletServerStatus> currentView = currentGrouped.get(regexTableName);
        if (currentView == null) {
            LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
            continue;
        }
        ArrayList<TabletMigration> newMigrations = new ArrayList<>();
        getBalancerForTable(tableId).balance(currentView, migrations, newMigrations);
        if (newMigrations.isEmpty()) {
            tableToTimeSinceNoMigrations.remove(s);
        } else if (tableToTimeSinceNoMigrations.containsKey(s)) {
            if ((now - tableToTimeSinceNoMigrations.get(s)) > HOURS.toMillis(1)) {
                LOG.warn("We have been consistently producing migrations for {}: {}", tableName, Iterables.limit(newMigrations, 10));
            }
        } else {
            tableToTimeSinceNoMigrations.put(s, now);
        }
        migrationsOut.addAll(newMigrations);
        if (migrationsOut.size() >= myConf.maxTServerMigrations) {
            break;
        }
    }
    for (TabletMigration migration : migrationsOut) {
        migrationsFromLastPass.put(migration.tablet, migration);
    }
    LOG.info("Migrating tablets for balance: {}", migrationsOut);
    return minBalanceTime;
}