Examples with ClusterMetrics org.apache.hadoop.hbase.ClusterMetrics used on opensource projects

Example 1 with ClusterMetrics

use of org.apache.hadoop.hbase.ClusterMetrics in project hbase by apache.

the class StorageClusterStatusResource method get.

@GET
@Produces({ MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF })
public Response get(@Context final UriInfo uriInfo) {
    if (LOG.isTraceEnabled()) {
        LOG.trace("GET " + uriInfo.getAbsolutePath());
    }
    servlet.getMetrics().incrementRequests(1);
    try {
        ClusterMetrics status = servlet.getAdmin().getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS, Option.DEAD_SERVERS));
        StorageClusterStatusModel model = new StorageClusterStatusModel();
        model.setRegions(status.getRegionCount());
        model.setRequests(status.getRequestCount());
        model.setAverageLoad(status.getAverageLoad());
        for (Map.Entry<ServerName, ServerMetrics> entry : status.getLiveServerMetrics().entrySet()) {
            ServerName sn = entry.getKey();
            ServerMetrics load = entry.getValue();
            StorageClusterStatusModel.Node node = model.addLiveNode(sn.getHostname() + ":" + Integer.toString(sn.getPort()), sn.getStartcode(), (int) load.getUsedHeapSize().get(Size.Unit.MEGABYTE), (int) load.getMaxHeapSize().get(Size.Unit.MEGABYTE));
            node.setRequests(load.getRequestCount());
            for (RegionMetrics region : load.getRegionMetrics().values()) {
                node.addRegion(region.getRegionName(), region.getStoreCount(), region.getStoreFileCount(), (int) region.getStoreFileSize().get(Size.Unit.MEGABYTE), (int) region.getMemStoreSize().get(Size.Unit.MEGABYTE), (long) region.getStoreFileIndexSize().get(Size.Unit.KILOBYTE), region.getReadRequestCount(), region.getCpRequestCount(), region.getWriteRequestCount(), (int) region.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE), (int) region.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE), (int) region.getBloomFilterSize().get(Size.Unit.KILOBYTE), region.getCompactingCellCount(), region.getCompactedCellCount());
            }
        }
        for (ServerName name : status.getDeadServerNames()) {
            model.addDeadNode(name.toString());
        }
        ResponseBuilder response = Response.ok(model);
        response.cacheControl(cacheControl);
        servlet.getMetrics().incrementSucessfulGetRequests(1);
        return response.build();
    } catch (IOException e) {
        servlet.getMetrics().incrementFailedGetRequests(1);
        return Response.status(Response.Status.SERVICE_UNAVAILABLE).type(MIMETYPE_TEXT).entity("Unavailable" + CRLF).build();
    }
}

Example 2 with ClusterMetrics

use of org.apache.hadoop.hbase.ClusterMetrics in project hbase by apache.

the class RegionsRecoveryChore method chore.

@Override
protected void chore() {
    if (LOG.isTraceEnabled()) {
        LOG.trace("Starting up Regions Recovery chore for reopening regions based on storeFileRefCount...");
    }
    try {
        // only if storeFileRefCountThreshold > 0, consider the feature turned on
        if (storeFileRefCountThreshold > 0) {
            final ClusterMetrics clusterMetrics = hMaster.getClusterMetrics();
            final Map<ServerName, ServerMetrics> serverMetricsMap = clusterMetrics.getLiveServerMetrics();
            final Map<TableName, List<byte[]>> tableToReopenRegionsMap = getTableToRegionsByRefCount(serverMetricsMap);
            if (MapUtils.isNotEmpty(tableToReopenRegionsMap)) {
                tableToReopenRegionsMap.forEach((tableName, regionNames) -> {
                    try {
                        LOG.warn("Reopening regions due to high storeFileRefCount. " + "TableName: {} , noOfRegions: {}", tableName, regionNames.size());
                        hMaster.reopenRegions(tableName, regionNames, NONCE_GENERATOR.getNonceGroup(), NONCE_GENERATOR.newNonce());
                    } catch (IOException e) {
                        LOG.error("{} tableName: {}, regionNames: {}", ERROR_REOPEN_REIONS_MSG, tableName, regionNames, e);
                    }
                });
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Reopening regions with very high storeFileRefCount is disabled. " + "Provide threshold value > 0 for {} to enable it.", HConstants.STORE_FILE_REF_COUNT_THRESHOLD);
            }
        }
    } catch (Exception e) {
        LOG.error("Error while reopening regions based on storeRefCount threshold", e);
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("Exiting Regions Recovery chore for reopening regions based on storeFileRefCount...");
    }
}

Example 3 with ClusterMetrics

use of org.apache.hadoop.hbase.ClusterMetrics in project hbase by apache.

the class HMaster method getClusterMetricsWithoutCoprocessor.

public ClusterMetrics getClusterMetricsWithoutCoprocessor(EnumSet<Option> options) throws InterruptedIOException {
    ClusterMetricsBuilder builder = ClusterMetricsBuilder.newBuilder();
    // we return all information to client if the list of Option is empty.
    if (options.isEmpty()) {
        options = EnumSet.allOf(Option.class);
    }
    // TASKS and/or LIVE_SERVERS will populate this map, which will be given to the builder if
    // not null after option processing completes.
    Map<ServerName, ServerMetrics> serverMetricsMap = null;
    for (Option opt : options) {
        switch(opt) {
            case HBASE_VERSION:
                builder.setHBaseVersion(VersionInfo.getVersion());
                break;
            case CLUSTER_ID:
                builder.setClusterId(getClusterId());
                break;
            case MASTER:
                builder.setMasterName(getServerName());
                break;
            case BACKUP_MASTERS:
                builder.setBackerMasterNames(getBackupMasters());
                break;
            case TASKS:
                {
                    // Master tasks
                    builder.setMasterTasks(TaskMonitor.get().getTasks().stream().map(task -> ServerTaskBuilder.newBuilder().setDescription(task.getDescription()).setStatus(task.getStatus()).setState(ServerTask.State.valueOf(task.getState().name())).setStartTime(task.getStartTime()).setCompletionTime(task.getCompletionTimestamp()).build()).collect(Collectors.toList()));
                    // Add entries to serverMetricsMap for all live servers, if we haven't already done so
                    if (serverMetricsMap == null) {
                        serverMetricsMap = getOnlineServers();
                    }
                    break;
                }
            case LIVE_SERVERS:
                {
                    // Add entries to serverMetricsMap for all live servers, if we haven't already done so
                    if (serverMetricsMap == null) {
                        serverMetricsMap = getOnlineServers();
                    }
                    break;
                }
            case DEAD_SERVERS:
                {
                    if (serverManager != null) {
                        builder.setDeadServerNames(new ArrayList<>(serverManager.getDeadServers().copyServerNames()));
                    }
                    break;
                }
            case MASTER_COPROCESSORS:
                {
                    if (cpHost != null) {
                        builder.setMasterCoprocessorNames(Arrays.asList(getMasterCoprocessors()));
                    }
                    break;
                }
            case REGIONS_IN_TRANSITION:
                {
                    if (assignmentManager != null) {
                        builder.setRegionsInTransition(assignmentManager.getRegionStates().getRegionsStateInTransition());
                    }
                    break;
                }
            case BALANCER_ON:
                {
                    if (loadBalancerTracker != null) {
                        builder.setBalancerOn(loadBalancerTracker.isBalancerOn());
                    }
                    break;
                }
            case MASTER_INFO_PORT:
                {
                    if (infoServer != null) {
                        builder.setMasterInfoPort(infoServer.getPort());
                    }
                    break;
                }
            case SERVERS_NAME:
                {
                    if (serverManager != null) {
                        builder.setServerNames(serverManager.getOnlineServersList());
                    }
                    break;
                }
            case TABLE_TO_REGIONS_COUNT:
                {
                    if (isActiveMaster() && isInitialized() && assignmentManager != null) {
                        try {
                            Map<TableName, RegionStatesCount> tableRegionStatesCountMap = new HashMap<>();
                            Map<String, TableDescriptor> tableDescriptorMap = getTableDescriptors().getAll();
                            for (TableDescriptor tableDescriptor : tableDescriptorMap.values()) {
                                TableName tableName = tableDescriptor.getTableName();
                                RegionStatesCount regionStatesCount = assignmentManager.getRegionStatesCount(tableName);
                                tableRegionStatesCountMap.put(tableName, regionStatesCount);
                            }
                            builder.setTableRegionStatesCount(tableRegionStatesCountMap);
                        } catch (IOException e) {
                            LOG.error("Error while populating TABLE_TO_REGIONS_COUNT for Cluster Metrics..", e);
                        }
                    }
                    break;
                }
        }
    }
    if (serverMetricsMap != null) {
        builder.setLiveServerMetrics(serverMetricsMap);
    }
    return builder.build();
}

Example 4 with ClusterMetrics

use of org.apache.hadoop.hbase.ClusterMetrics in project hbase by apache.

the class RawAsyncHBaseAdmin method getRegionServerList.

private CompletableFuture<List<ServerName>> getRegionServerList(List<String> serverNamesList) {
    CompletableFuture<List<ServerName>> future = new CompletableFuture<>();
    if (serverNamesList.isEmpty()) {
        CompletableFuture<ClusterMetrics> clusterMetricsCompletableFuture = getClusterMetrics(EnumSet.of(Option.SERVERS_NAME));
        addListener(clusterMetricsCompletableFuture, (clusterMetrics, err) -> {
            if (err != null) {
                future.completeExceptionally(err);
            } else {
                future.complete(clusterMetrics.getServersName());
            }
        });
        return future;
    } else {
        List<ServerName> serverList = new ArrayList<>();
        for (String regionServerName : serverNamesList) {
            ServerName serverName = null;
            try {
                serverName = ServerName.valueOf(regionServerName);
            } catch (Exception e) {
                future.completeExceptionally(new IllegalArgumentException(String.format("ServerName format: %s", regionServerName)));
            }
            if (serverName == null) {
                future.completeExceptionally(new IllegalArgumentException(String.format("Null ServerName: %s", regionServerName)));
            } else {
                serverList.add(serverName);
            }
        }
        future.complete(serverList);
    }
    return future;
}

Example 5 with ClusterMetrics

use of org.apache.hadoop.hbase.ClusterMetrics in project hbase by apache.

the class UnbalanceKillAndRebalanceAction method perform.

@Override
public void perform() throws Exception {
    ClusterMetrics status = this.cluster.getClusterMetrics();
    List<ServerName> victimServers = new LinkedList<>(status.getLiveServerMetrics().keySet());
    Set<ServerName> killedServers = new HashSet<>();
    int liveCount = (int) Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size());
    int deadCount = (int) Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size());
    Assert.assertTrue("There are not enough victim servers: " + victimServers.size(), liveCount + deadCount < victimServers.size());
    List<ServerName> targetServers = new ArrayList<>(liveCount);
    for (int i = 0; i < liveCount + deadCount; ++i) {
        int victimIx = RandomUtils.nextInt(0, victimServers.size());
        targetServers.add(victimServers.remove(victimIx));
    }
    unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS);
    Thread.sleep(waitForUnbalanceMilliSec);
    ServerName metaServer = cluster.getServerHoldingMeta();
    for (ServerName targetServer : targetServers) {
        // trying to stop the monkey.
        if (context.isStopping()) {
            break;
        }
        if (killedServers.size() >= liveCount) {
            break;
        }
        if (!killMetaRs && targetServer.equals(metaServer)) {
            getLogger().info("Not killing server because it holds hbase:meta.");
        } else {
            killRs(targetServer);
            killedServers.add(targetServer);
        }
    }
    Thread.sleep(waitForKillsMilliSec);
    forceBalancer();
    Thread.sleep(waitAfterBalanceMilliSec);
    for (ServerName server : killedServers) {
        startRs(server);
    }
}