Examples with TabletServerStatus org.apache.accumulo.core.master.thrift.TabletServerStatus used on opensource projects

Example 6 with TabletServerStatus

use of org.apache.accumulo.core.master.thrift.TabletServerStatus in project accumulo by apache.

the class TableLoadBalancerTest method generateFakeTablets.

static List<TabletStats> generateFakeTablets(TServerInstance tserver, Table.ID tableId) {
    List<TabletStats> result = new ArrayList<>();
    TabletServerStatus tableInfo = state.get(tserver);
    // generate some fake tablets
    for (int i = 0; i < tableInfo.tableMap.get(tableId.canonicalID()).onlineTablets; i++) {
        TabletStats stats = new TabletStats();
        stats.extent = new KeyExtent(tableId, new Text(tserver.host() + String.format("%03d", i + 1)), new Text(tserver.host() + String.format("%03d", i))).toThrift();
        result.add(stats);
    }
    return result;
}

Example 7 with TabletServerStatus

use of org.apache.accumulo.core.master.thrift.TabletServerStatus in project accumulo by apache.

the class ChaoticLoadBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    Map<TServerInstance, Long> numTablets = new HashMap<>();
    List<TServerInstance> underCapacityTServer = new ArrayList<>();
    if (!migrations.isEmpty()) {
        outstandingMigrations.migrations = migrations;
        constraintNotMet(outstandingMigrations);
        return 100;
    }
    resetBalancerErrors();
    boolean moveMetadata = r.nextInt(4) == 0;
    long totalTablets = 0;
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        long tabletCount = 0;
        for (TableInfo ti : e.getValue().getTableMap().values()) {
            tabletCount += ti.tablets;
        }
        numTablets.put(e.getKey(), tabletCount);
        underCapacityTServer.add(e.getKey());
        totalTablets += tabletCount;
    }
    // totalTablets is fuzzy due to asynchronicity of the stats
    // *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing scenarios
    long avg = (long) Math.ceil(((double) totalTablets) / current.size() * 1.2);
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        for (String tableId : e.getValue().getTableMap().keySet()) {
            Table.ID id = Table.ID.of(tableId);
            if (!moveMetadata && MetadataTable.ID.equals(id))
                continue;
            try {
                for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), id)) {
                    KeyExtent ke = new KeyExtent(ts.extent);
                    int index = r.nextInt(underCapacityTServer.size());
                    TServerInstance dest = underCapacityTServer.get(index);
                    if (dest.equals(e.getKey()))
                        continue;
                    migrationsOut.add(new TabletMigration(ke, e.getKey(), dest));
                    if (numTablets.put(dest, numTablets.get(dest) + 1) > avg)
                        underCapacityTServer.remove(index);
                    if (numTablets.put(e.getKey(), numTablets.get(e.getKey()) - 1) <= avg && !underCapacityTServer.contains(e.getKey()))
                        underCapacityTServer.add(e.getKey());
                    // We can get some craziness with only 1 tserver, so lets make sure there's always an option!
                    if (underCapacityTServer.isEmpty())
                        underCapacityTServer.addAll(numTablets.keySet());
                }
            } catch (ThriftSecurityException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered ThriftSecurityException.  This should not happen.  Carrying on anyway.", e1);
            } catch (TException e1) {
                // Shouldn't happen, but carry on if it does
                log.debug("Encountered TException.  This should not happen.  Carrying on anyway.", e1);
            }
        }
    }
    return 100;
}

Example 8 with TabletServerStatus

use of org.apache.accumulo.core.master.thrift.TabletServerStatus in project accumulo by apache.

the class ChaoticLoadBalancer method getAssignments.

@Override
public void getAssignments(SortedMap<TServerInstance, TabletServerStatus> current, Map<KeyExtent, TServerInstance> unassigned, Map<KeyExtent, TServerInstance> assignments) {
    long total = assignments.size() + unassigned.size();
    long avg = (long) Math.ceil(((double) total) / current.size());
    Map<TServerInstance, Long> toAssign = new HashMap<>();
    List<TServerInstance> tServerArray = new ArrayList<>();
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        long numTablets = 0;
        for (TableInfo ti : e.getValue().getTableMap().values()) {
            numTablets += ti.tablets;
        }
        if (numTablets <= avg) {
            tServerArray.add(e.getKey());
            toAssign.put(e.getKey(), avg - numTablets);
        }
    }
    if (tServerArray.isEmpty()) {
        // No tservers to assign to
        return;
    }
    for (KeyExtent ke : unassigned.keySet()) {
        int index = r.nextInt(tServerArray.size());
        TServerInstance dest = tServerArray.get(index);
        assignments.put(ke, dest);
        long remaining = toAssign.get(dest) - 1;
        if (remaining == 0) {
            tServerArray.remove(index);
            toAssign.remove(dest);
        } else {
            toAssign.put(dest, remaining);
        }
    }
}

Example 9 with TabletServerStatus

use of org.apache.accumulo.core.master.thrift.TabletServerStatus in project accumulo by apache.

the class TabletGroupWatcher method run.

@Override
public void run() {
    Thread.currentThread().setName("Watching " + store.name());
    int[] oldCounts = new int[TabletState.values().length];
    EventCoordinator.Listener eventListener = this.master.nextEvent.getListener();
    WalStateManager wals = new WalStateManager(master.getInstance(), ZooReaderWriter.getInstance());
    while (this.master.stillMaster()) {
        // slow things down a little, otherwise we spam the logs when there are many wake-up events
        sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        masterState = master.getMasterState();
        int totalUnloaded = 0;
        int unloaded = 0;
        ClosableIterator<TabletLocationState> iter = null;
        try {
            Map<Table.ID, MergeStats> mergeStatsCache = new HashMap<>();
            Map<Table.ID, MergeStats> currentMerges = new HashMap<>();
            for (MergeInfo merge : master.merges()) {
                if (merge.getExtent() != null) {
                    currentMerges.put(merge.getExtent().getTableId(), new MergeStats(merge));
                }
            }
            // Get the current status for the current list of tservers
            SortedMap<TServerInstance, TabletServerStatus> currentTServers = new TreeMap<>();
            for (TServerInstance entry : this.master.tserverSet.getCurrentServers()) {
                currentTServers.put(entry, this.master.tserverStatus.get(entry));
            }
            if (currentTServers.size() == 0) {
                eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
                synchronized (this) {
                    lastScanServers = ImmutableSortedSet.of();
                }
                continue;
            }
            // Don't move tablets to servers that are shutting down
            SortedMap<TServerInstance, TabletServerStatus> destinations = new TreeMap<>(currentTServers);
            destinations.keySet().removeAll(this.master.serversToShutdown);
            List<Assignment> assignments = new ArrayList<>();
            List<Assignment> assigned = new ArrayList<>();
            List<TabletLocationState> assignedToDeadServers = new ArrayList<>();
            List<TabletLocationState> suspendedToGoneServers = new ArrayList<>();
            Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
            Map<TServerInstance, List<Path>> logsForDeadServers = new TreeMap<>();
            MasterState masterState = master.getMasterState();
            int[] counts = new int[TabletState.values().length];
            stats.begin();
            // Walk through the tablets in our store, and work tablets
            // towards their goal
            iter = store.iterator();
            while (iter.hasNext()) {
                TabletLocationState tls = iter.next();
                if (tls == null) {
                    continue;
                }
                Master.log.debug("{} location State: {}", store.name(), tls);
                // ignore entries for tables that do not exist in zookeeper
                if (TableManager.getInstance().getTableState(tls.extent.getTableId()) == null)
                    continue;
                if (Master.log.isTraceEnabled())
                    Master.log.trace("{} walogs {}", tls, tls.walogs.size());
                // Don't overwhelm the tablet servers with work
                if (unassigned.size() + unloaded > Master.MAX_TSERVER_WORK_CHUNK * currentTServers.size()) {
                    flushChanges(destinations, assignments, assigned, assignedToDeadServers, logsForDeadServers, suspendedToGoneServers, unassigned);
                    assignments.clear();
                    assigned.clear();
                    assignedToDeadServers.clear();
                    suspendedToGoneServers.clear();
                    unassigned.clear();
                    unloaded = 0;
                    eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
                }
                Table.ID tableId = tls.extent.getTableId();
                TableConfiguration tableConf = this.master.getConfigurationFactory().getTableConfiguration(tableId);
                MergeStats mergeStats = mergeStatsCache.get(tableId);
                if (mergeStats == null) {
                    mergeStats = currentMerges.get(tableId);
                    if (mergeStats == null) {
                        mergeStats = new MergeStats(new MergeInfo());
                    }
                    mergeStatsCache.put(tableId, mergeStats);
                }
                TabletGoalState goal = this.master.getGoalState(tls, mergeStats.getMergeInfo());
                TServerInstance server = tls.getServer();
                TabletState state = tls.getState(currentTServers.keySet());
                if (Master.log.isTraceEnabled()) {
                    Master.log.trace("Goal state {} current {} for {}", goal, state, tls.extent);
                }
                stats.update(tableId, state);
                mergeStats.update(tls.extent, state, tls.chopped, !tls.walogs.isEmpty());
                sendChopRequest(mergeStats.getMergeInfo(), state, tls);
                sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
                // Always follow through with assignments
                if (state == TabletState.ASSIGNED) {
                    goal = TabletGoalState.HOSTED;
                }
                // if we are shutting down all the tabletservers, we have to do it in order
                if (goal == TabletGoalState.SUSPENDED && state == TabletState.HOSTED) {
                    if (this.master.serversToShutdown.equals(currentTServers.keySet())) {
                        if (dependentWatcher != null && dependentWatcher.assignedOrHosted() > 0) {
                            goal = TabletGoalState.HOSTED;
                        }
                    }
                }
                if (goal == TabletGoalState.HOSTED) {
                    if (state != TabletState.HOSTED && !tls.walogs.isEmpty()) {
                        if (this.master.recoveryManager.recoverLogs(tls.extent, tls.walogs))
                            continue;
                    }
                    switch(state) {
                        case HOSTED:
                            if (server.equals(this.master.migrations.get(tls.extent)))
                                this.master.migrations.remove(tls.extent);
                            break;
                        case ASSIGNED_TO_DEAD_SERVER:
                            assignedToDeadServers.add(tls);
                            if (server.equals(this.master.migrations.get(tls.extent)))
                                this.master.migrations.remove(tls.extent);
                            TServerInstance tserver = tls.futureOrCurrent();
                            if (!logsForDeadServers.containsKey(tserver)) {
                                logsForDeadServers.put(tserver, wals.getWalsInUse(tserver));
                            }
                            break;
                        case SUSPENDED:
                            if (master.getSteadyTime() - tls.suspend.suspensionTime < tableConf.getTimeInMillis(Property.TABLE_SUSPEND_DURATION)) {
                                // Tablet is suspended. See if its tablet server is back.
                                TServerInstance returnInstance = null;
                                Iterator<TServerInstance> find = destinations.tailMap(new TServerInstance(tls.suspend.server, " ")).keySet().iterator();
                                if (find.hasNext()) {
                                    TServerInstance found = find.next();
                                    if (found.getLocation().equals(tls.suspend.server)) {
                                        returnInstance = found;
                                    }
                                }
                                // Old tablet server is back. Return this tablet to its previous owner.
                                if (returnInstance != null) {
                                    assignments.add(new Assignment(tls.extent, returnInstance));
                                } else {
                                // leave suspended, don't ask for a new assignment.
                                }
                            } else {
                                // Treat as unassigned, ask for a new assignment.
                                unassigned.put(tls.extent, server);
                            }
                            break;
                        case UNASSIGNED:
                            // maybe it's a finishing migration
                            TServerInstance dest = this.master.migrations.get(tls.extent);
                            if (dest != null) {
                                // if destination is still good, assign it
                                if (destinations.keySet().contains(dest)) {
                                    assignments.add(new Assignment(tls.extent, dest));
                                } else {
                                    // get rid of this migration
                                    this.master.migrations.remove(tls.extent);
                                    unassigned.put(tls.extent, server);
                                }
                            } else {
                                unassigned.put(tls.extent, server);
                            }
                            break;
                        case ASSIGNED:
                            // Send another reminder
                            assigned.add(new Assignment(tls.extent, tls.future));
                            break;
                    }
                } else {
                    switch(state) {
                        case SUSPENDED:
                            // Request a move to UNASSIGNED, so as to allow balancing to continue.
                            suspendedToGoneServers.add(tls);
                            cancelOfflineTableMigrations(tls);
                            break;
                        case UNASSIGNED:
                            cancelOfflineTableMigrations(tls);
                            break;
                        case ASSIGNED_TO_DEAD_SERVER:
                            assignedToDeadServers.add(tls);
                            if (!logsForDeadServers.containsKey(tls.futureOrCurrent())) {
                                logsForDeadServers.put(tls.futureOrCurrent(), wals.getWalsInUse(tls.futureOrCurrent()));
                            }
                            break;
                        case HOSTED:
                            TServerConnection conn = this.master.tserverSet.getConnection(server);
                            if (conn != null) {
                                conn.unloadTablet(this.master.masterLock, tls.extent, goal.howUnload(), master.getSteadyTime());
                                unloaded++;
                                totalUnloaded++;
                            } else {
                                Master.log.warn("Could not connect to server {}", server);
                            }
                            break;
                        case ASSIGNED:
                            break;
                    }
                }
                counts[state.ordinal()]++;
            }
            flushChanges(destinations, assignments, assigned, assignedToDeadServers, logsForDeadServers, suspendedToGoneServers, unassigned);
            // provide stats after flushing changes to avoid race conditions w/ delete table
            stats.end(masterState);
            // Report changes
            for (TabletState state : TabletState.values()) {
                int i = state.ordinal();
                if (counts[i] > 0 && counts[i] != oldCounts[i]) {
                    this.master.nextEvent.event("[%s]: %d tablets are %s", store.name(), counts[i], state.name());
                }
            }
            Master.log.debug(String.format("[%s]: scan time %.2f seconds", store.name(), stats.getScanTime() / 1000.));
            oldCounts = counts;
            if (totalUnloaded > 0) {
                this.master.nextEvent.event("[%s]: %d tablets unloaded", store.name(), totalUnloaded);
            }
            updateMergeState(mergeStatsCache);
            synchronized (this) {
                lastScanServers = ImmutableSortedSet.copyOf(currentTServers.keySet());
            }
            if (this.master.tserverSet.getCurrentServers().equals(currentTServers.keySet())) {
                Master.log.debug(String.format("[%s] sleeping for %.2f seconds", store.name(), Master.TIME_TO_WAIT_BETWEEN_SCANS / 1000.));
                eventListener.waitForEvents(Master.TIME_TO_WAIT_BETWEEN_SCANS);
            } else {
                Master.log.info("Detected change in current tserver set, re-running state machine.");
            }
        } catch (Exception ex) {
            Master.log.error("Error processing table state for store " + store.name(), ex);
            if (ex.getCause() != null && ex.getCause() instanceof BadLocationStateException) {
                repairMetadata(((BadLocationStateException) ex.getCause()).getEncodedEndRow());
            } else {
                sleepUninterruptibly(Master.WAIT_BETWEEN_ERRORS, TimeUnit.MILLISECONDS);
            }
        } finally {
            if (iter != null) {
                try {
                    iter.close();
                } catch (IOException ex) {
                    Master.log.warn("Error closing TabletLocationState iterator: " + ex, ex);
                }
            }
        }
    }
}

Example 10 with TabletServerStatus

use of org.apache.accumulo.core.master.thrift.TabletServerStatus in project accumulo by apache.

the class Monitor method fetchData.

public static void fetchData() {
    double totalIngestRate = 0.;
    double totalIngestByteRate = 0.;
    double totalQueryRate = 0.;
    double totalQueryByteRate = 0.;
    double totalScanRate = 0.;
    long totalEntries = 0;
    int totalTabletCount = 0;
    long totalHoldTime = 0;
    long totalLookups = 0;
    boolean retry = true;
    // only recalc every so often
    long currentTime = System.currentTimeMillis();
    if (currentTime - lastRecalc.get() < REFRESH_TIME * 1000)
        return;
    synchronized (Monitor.class) {
        // Learn our instance name asynchronously so we don't hang up if zookeeper is down
        if (cachedInstanceName.get().equals(DEFAULT_INSTANCE_NAME)) {
            SimpleTimer.getInstance(config.getSystemConfiguration()).schedule(new TimerTask() {

                @Override
                public void run() {
                    synchronized (Monitor.class) {
                        if (cachedInstanceName.get().equals(DEFAULT_INSTANCE_NAME)) {
                            final String instanceName = HdfsZooInstance.getInstance().getInstanceName();
                            if (null != instanceName) {
                                cachedInstanceName.set(instanceName);
                            }
                        }
                    }
                }
            }, 0);
        }
    }
    synchronized (Monitor.class) {
        if (fetching)
            return;
        fetching = true;
    }
    try {
        while (retry) {
            MasterClientService.Iface client = null;
            try {
                client = MasterClient.getConnection(context);
                if (client != null) {
                    mmi = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
                    retry = false;
                } else {
                    mmi = null;
                }
                Monitor.gcStatus = fetchGcStatus();
            } catch (Exception e) {
                mmi = null;
                log.info("Error fetching stats: ", e);
            } finally {
                if (client != null) {
                    MasterClient.close(client);
                }
            }
            if (mmi == null)
                sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
        if (mmi != null) {
            int majorCompactions = 0;
            int minorCompactions = 0;
            lookupRateTracker.startingUpdates();
            indexCacheHitTracker.startingUpdates();
            indexCacheRequestTracker.startingUpdates();
            dataCacheHitTracker.startingUpdates();
            dataCacheRequestTracker.startingUpdates();
            for (TabletServerStatus server : mmi.tServerInfo) {
                TableInfo summary = TableInfoUtil.summarizeTableStats(server);
                totalIngestRate += summary.ingestRate;
                totalIngestByteRate += summary.ingestByteRate;
                totalQueryRate += summary.queryRate;
                totalScanRate += summary.scanRate;
                totalQueryByteRate += summary.queryByteRate;
                totalEntries += summary.recs;
                totalHoldTime += server.holdTime;
                totalLookups += server.lookups;
                majorCompactions += summary.majors.running;
                minorCompactions += summary.minors.running;
                lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
                indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
                indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
                dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
                dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
            }
            lookupRateTracker.finishedUpdating();
            indexCacheHitTracker.finishedUpdating();
            indexCacheRequestTracker.finishedUpdating();
            dataCacheHitTracker.finishedUpdating();
            dataCacheRequestTracker.finishedUpdating();
            int totalTables = 0;
            for (TableInfo tInfo : mmi.tableMap.values()) {
                totalTabletCount += tInfo.tablets;
                totalTables++;
            }
            Monitor.totalIngestRate = totalIngestRate;
            Monitor.totalTables = totalTables;
            totalIngestByteRate = totalIngestByteRate / 1000000.0;
            Monitor.totalQueryRate = totalQueryRate;
            Monitor.totalScanRate = totalScanRate;
            totalQueryByteRate = totalQueryByteRate / 1000000.0;
            Monitor.totalEntries = totalEntries;
            Monitor.totalTabletCount = totalTabletCount;
            Monitor.totalHoldTime = totalHoldTime;
            Monitor.totalLookups = totalLookups;
            ingestRateOverTime.add(new Pair<>(currentTime, totalIngestRate));
            ingestByteRateOverTime.add(new Pair<>(currentTime, totalIngestByteRate));
            double totalLoad = 0.;
            for (TabletServerStatus status : mmi.tServerInfo) {
                if (status != null)
                    totalLoad += status.osLoad;
            }
            loadOverTime.add(new Pair<>(currentTime, totalLoad));
            minorCompactionsOverTime.add(new Pair<>(currentTime, minorCompactions));
            majorCompactionsOverTime.add(new Pair<>(currentTime, majorCompactions));
            lookupsOverTime.add(new Pair<>(currentTime, lookupRateTracker.calculateRate()));
            queryRateOverTime.add(new Pair<>(currentTime, (int) totalQueryRate));
            queryByteRateOverTime.add(new Pair<>(currentTime, totalQueryByteRate));
            scanRateOverTime.add(new Pair<>(currentTime, (int) totalScanRate));
            calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
            calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
        }
        try {
            Monitor.problemSummary = ProblemReports.getInstance(getContext()).summarize();
            Monitor.problemException = null;
        } catch (Exception e) {
            log.info("Failed to obtain problem reports ", e);
            Monitor.problemSummary = Collections.emptyMap();
            Monitor.problemException = e;
        }
    } finally {
        synchronized (Monitor.class) {
            fetching = false;
            lastRecalc.set(currentTime);
        }
    }
}