Search in sources :

Example 6 with HostPartitionClassStorageStatsWrapper

use of com.github.ambry.server.HostPartitionClassStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStore method queryHostPartitionClassStorageStatsByHost.

@Override
public HostPartitionClassStorageStatsWrapper queryHostPartitionClassStorageStatsByHost(String hostname, int port, Map<String, Set<Integer>> partitionNameAndIds) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    hostname = hostnameHelper.simplifyHostname(hostname, port);
    Map<Integer, Map<Short, Map<Short, ContainerStorageStats>>> partitionAccountContainerUsage = new HashMap<>();
    AtomicLong timestamp = new AtomicLong(0);
    accountReportsDao.queryStorageUsageForHost(clusterName, hostname, (partitionId, accountId, containerStats, updatedAtMs) -> {
        partitionAccountContainerUsage.computeIfAbsent(partitionId, pid -> new HashMap<>()).computeIfAbsent(accountId, aid -> new HashMap<>()).put(containerStats.getContainerId(), containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAtMs));
    });
    // Here partitionAccountContainerUsage has partition id, account id and container id as keys of map at each level,
    // the value is the storage usage.
    // As indicated by the comments above, we have to know the partition class name for each partition id. Luckily, we
    // have all the partition ids and we have a map partitionNameAndIds whose key is the partition class name and the
    // value is the list of all partition ids belong to the partition class name.
    Set<Integer> partitionIds = partitionAccountContainerUsage.keySet();
    Map<String, Set<Integer>> partitionNameAndIdsForHost = new HashMap<>();
    for (int partitionId : partitionIds) {
        boolean found = false;
        for (Map.Entry<String, Set<Integer>> namesAndIdsEntry : partitionNameAndIds.entrySet()) {
            if (namesAndIdsEntry.getValue().contains(partitionId)) {
                partitionNameAndIdsForHost.computeIfAbsent(namesAndIdsEntry.getKey(), k -> new HashSet<>()).add(partitionId);
                found = true;
                break;
            }
        }
        if (!found) {
            storeMetrics.missingPartitionClassNameErrorCount.inc();
            logger.error("Can't find partition class name for partition id {}", partitionId);
        }
    }
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    for (Map.Entry<String, Set<Integer>> nameAndIdsEntry : partitionNameAndIdsForHost.entrySet()) {
        String partitionClassName = nameAndIdsEntry.getKey();
        for (int partitionId : nameAndIdsEntry.getValue()) {
            Map<Short, Map<Short, ContainerStorageStats>> accountContainerUsage = partitionAccountContainerUsage.get(partitionId);
            for (short accountId : accountContainerUsage.keySet()) {
                Map<Short, ContainerStorageStats> containerUsage = accountContainerUsage.get(accountId);
                containerUsage.values().stream().forEach(containerStats -> hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, containerStats));
            }
        }
    }
    storeMetrics.queryPartitionClassStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return new HostPartitionClassStorageStatsWrapper(new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp.get(), partitionAccountContainerUsage.size(), partitionAccountContainerUsage.size(), null), hostPartitionClassStorageStats);
}
Also used : HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) Histogram(com.codahale.metrics.Histogram) HostAccountStorageStatsWrapper(com.github.ambry.server.HostAccountStorageStatsWrapper) Connection(java.sql.Connection) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) StatsHeader(com.github.ambry.server.StatsHeader) Strings(joptsimple.internal.Strings) LoggerFactory(org.slf4j.LoggerFactory) HostAccountStorageStats(com.github.ambry.server.storagestats.HostAccountStorageStats) HashMap(java.util.HashMap) ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) SQLException(java.sql.SQLException) Map(java.util.Map) Counter(com.codahale.metrics.Counter) DataSource(javax.sql.DataSource) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Set(java.util.Set) Utils(com.github.ambry.utils.Utils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) File(java.io.File) AccountStatsMySqlConfig(com.github.ambry.config.AccountStatsMySqlConfig) AggregatedAccountStorageStats(com.github.ambry.server.storagestats.AggregatedAccountStorageStats) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) MySqlMetrics(com.github.ambry.mysql.MySqlMetrics) Statement(java.sql.Statement) Optional(java.util.Optional) SerializationFeature(com.fasterxml.jackson.databind.SerializationFeature) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) HashMap(java.util.HashMap) StatsHeader(com.github.ambry.server.StatsHeader) HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) AtomicLong(java.util.concurrent.atomic.AtomicLong) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Aggregations

HostPartitionClassStorageStatsWrapper (com.github.ambry.server.HostPartitionClassStorageStatsWrapper)6 HostPartitionClassStorageStats (com.github.ambry.server.storagestats.HostPartitionClassStorageStats)6 HashMap (java.util.HashMap)6 Map (java.util.Map)6 StatsHeader (com.github.ambry.server.StatsHeader)5 AggregatedPartitionClassStorageStats (com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats)5 ContainerStorageStats (com.github.ambry.server.storagestats.ContainerStorageStats)5 HostAccountStorageStatsWrapper (com.github.ambry.server.HostAccountStorageStatsWrapper)4 AggregatedAccountStorageStats (com.github.ambry.server.storagestats.AggregatedAccountStorageStats)4 HostAccountStorageStats (com.github.ambry.server.storagestats.HostAccountStorageStats)4 Pair (com.github.ambry.utils.Pair)4 Collections (java.util.Collections)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 StorageStatsUtilTest (com.github.ambry.server.StorageStatsUtilTest)3 Assert (org.junit.Assert)3 Test (org.junit.Test)3 MetricRegistry (com.codahale.metrics.MetricRegistry)2 TypeReference (com.fasterxml.jackson.core.type.TypeReference)2 AccountStatsMySqlConfig (com.github.ambry.config.AccountStatsMySqlConfig)2 Utils (com.github.ambry.utils.Utils)2