Search in sources :

Example 1 with HostPartitionClassStorageStats

use of com.github.ambry.server.storagestats.HostPartitionClassStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostPartitionClassStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostPartitionClassStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostPartitionClassStorageStats() throws Exception {
    int nodeCount = 3;
    int numberOfPartitions = 4;
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> storageStatsMap = new HashMap<>();
    String[] partitionClassNames = { "default", "newClass" };
    for (int i = 0; i < numberOfPartitions; i++) {
        String partitionClassName = partitionClassNames[i % partitionClassNames.length];
        storageStatsMap.computeIfAbsent(partitionClassName, k -> new HashMap<>()).put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, numberOfPartitions, numberOfPartitions, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper nodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper emptyStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats());
    Map<String, HostPartitionClassStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostPartitionClassStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostPartitionClassStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(instanceToStatsMap);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostPartitionClassStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForPartitionClassStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}
Also used : HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) HostAccountStorageStatsWrapper(com.github.ambry.server.HostAccountStorageStatsWrapper) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) Pair(com.github.ambry.utils.Pair) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) StatsHeader(com.github.ambry.server.StatsHeader) HostAccountStorageStats(com.github.ambry.server.storagestats.HostAccountStorageStats) HashMap(java.util.HashMap) Test(org.junit.Test) ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) LinkedHashMap(java.util.LinkedHashMap) TimeUnit(java.util.concurrent.TimeUnit) AggregatedAccountStorageStats(com.github.ambry.server.storagestats.AggregatedAccountStorageStats) Map(java.util.Map) TypeReference(com.fasterxml.jackson.core.type.TypeReference) Assert(org.junit.Assert) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) Collections(java.util.Collections) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) StatsHeader(com.github.ambry.server.StatsHeader) HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) Test(org.junit.Test) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest)

Example 2 with HostPartitionClassStorageStats

use of com.github.ambry.server.storagestats.HostPartitionClassStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testHostPartitionClassStorageStats.

/**
 * Test methods to store and fetch partition class, partition name partition id and partition class storage stats.
 * @throws Exception
 */
@Test
public void testHostPartitionClassStorageStats() throws Exception {
    // First write some stats to account reports
    testMultiStoreStats();
    HostAccountStorageStatsWrapper accountStats1 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    HostAccountStorageStatsWrapper accountStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname2, port2);
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    HostAccountStorageStatsWrapper accountStats3 = mySqlStore3.queryHostAccountStorageStatsByHost(hostname3, port3);
    // From this account stats, create partition class stats;
    Set<Long> allPartitionKeys = new HashSet<Long>() {

        {
            addAll(accountStats1.getStats().getStorageStats().keySet());
            addAll(accountStats2.getStats().getStorageStats().keySet());
            addAll(accountStats3.getStats().getStorageStats().keySet());
        }
    };
    List<String> partitionClassNames = Arrays.asList("default", "new");
    Map<Long, String> partitionIdToClassName = new HashMap<>();
    int ind = 0;
    for (long partitionId : allPartitionKeys) {
        partitionIdToClassName.put(partitionId, partitionClassNames.get(ind % partitionClassNames.size()));
        ind++;
    }
    HostPartitionClassStorageStatsWrapper partitionClassStats1 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats1, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats2 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats2, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats3 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats3, partitionIdToClassName);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats1);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats2);
    mySqlStore3.storeHostPartitionClassStorageStats(partitionClassStats3);
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    assertEquals(new HashSet<>(partitionClassNames), partitionNameAndIds.keySet());
    Map<Long, String> dbPartitionKeyToClassName = partitionNameAndIds.entrySet().stream().flatMap(ent -> ent.getValue().stream().map(pid -> new Pair<>(ent.getKey(), (long) pid))).collect(Collectors.toMap(Pair::getSecond, Pair::getFirst));
    assertEquals(partitionIdToClassName, dbPartitionKeyToClassName);
    // Fetch HostPartitionClassStorageStats
    HostPartitionClassStorageStatsWrapper obtainedStats1 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(partitionClassStats1.getStats().getStorageStats(), obtainedStats1.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname2, port2, partitionNameAndIds);
    assertEquals(partitionClassStats2.getStats().getStorageStats(), obtainedStats2.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats3 = mySqlStore3.queryHostPartitionClassStorageStatsByHost(hostname3, port3, partitionNameAndIds);
    assertEquals(partitionClassStats3.getStats().getStorageStats(), obtainedStats3.getStats().getStorageStats());
    // Fetch StatsSnapshot
    StatsWrapper obtainedStats = mySqlStore.queryPartitionClassStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(StorageStatsUtil.convertHostPartitionClassStorageStatsToStatsSnapshot(obtainedStats1.getStats(), false), obtainedStats.getSnapshot());
    mySqlStore3.shutdown();
}
Also used : HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) HostAccountStorageStatsWrapper(com.github.ambry.server.HostAccountStorageStatsWrapper) Arrays(java.util.Arrays) Connection(java.sql.Connection) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) StatsHeader(com.github.ambry.server.StatsHeader) RunWith(org.junit.runner.RunWith) HostAccountStorageStats(com.github.ambry.server.storagestats.HostAccountStorageStats) HashMap(java.util.HashMap) Random(java.util.Random) ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) SQLException(java.sql.SQLException) TestUtils(com.github.ambry.utils.TestUtils) ResultSet(java.sql.ResultSet) Map(java.util.Map) After(org.junit.After) Path(java.nio.file.Path) Parameterized(org.junit.runners.Parameterized) StatsWrapper(com.github.ambry.server.StatsWrapper) Before(org.junit.Before) MetricRegistry(com.codahale.metrics.MetricRegistry) Properties(java.util.Properties) Pair(com.github.ambry.utils.Pair) Files(java.nio.file.Files) VerifiableProperties(com.github.ambry.config.VerifiableProperties) Set(java.util.Set) Utils(com.github.ambry.utils.Utils) IOException(java.io.IOException) Test(org.junit.Test) Collectors(java.util.stream.Collectors) AccountStatsMySqlConfig(com.github.ambry.config.AccountStatsMySqlConfig) AggregatedAccountStorageStats(com.github.ambry.server.storagestats.AggregatedAccountStorageStats) List(java.util.List) StorageStatsUtil(com.github.ambry.server.StorageStatsUtil) StatsReportType(com.github.ambry.server.StatsReportType) StatsSnapshot(com.github.ambry.server.StatsSnapshot) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) Statement(java.sql.Statement) Assert(org.junit.Assert) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) Collections(java.util.Collections) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest) HostAccountStorageStatsWrapper(com.github.ambry.server.HostAccountStorageStatsWrapper) HashSet(java.util.HashSet) ResultSet(java.sql.ResultSet) Set(java.util.Set) HashMap(java.util.HashMap) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) HostAccountStorageStatsWrapper(com.github.ambry.server.HostAccountStorageStatsWrapper) StatsWrapper(com.github.ambry.server.StatsWrapper) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) HashSet(java.util.HashSet) Test(org.junit.Test) StorageStatsUtilTest(com.github.ambry.server.StorageStatsUtilTest)

Example 3 with HostPartitionClassStorageStats

use of com.github.ambry.server.storagestats.HostPartitionClassStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method convertHostAccountStorageStatsToHostPartitionClassStorageStats.

private HostPartitionClassStorageStatsWrapper convertHostAccountStorageStatsToHostPartitionClassStorageStats(HostAccountStorageStatsWrapper accountStatsWrapper, Map<Long, String> partitionIdToClassName) {
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = accountStatsWrapper.getStats().getStorageStats();
    for (long partitionId : storageStats.keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> accountStorageStatsMap = storageStats.get(partitionId);
        String partitionClassName = partitionIdToClassName.get(partitionId);
        for (short accountId : accountStorageStatsMap.keySet()) {
            accountStorageStatsMap.get(accountId).values().forEach(containerStats -> hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, containerStats));
        }
    }
    return new HostPartitionClassStorageStatsWrapper(new StatsHeader(accountStatsWrapper.getHeader()), hostPartitionClassStorageStats);
}
Also used : ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) StatsHeader(com.github.ambry.server.StatsHeader) HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) HashMap(java.util.HashMap) Map(java.util.Map)

Example 4 with HostPartitionClassStorageStats

use of com.github.ambry.server.storagestats.HostPartitionClassStorageStats in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostPartitionClassStorageStatsWrappers.

/**
 * Aggregate all {@link HostPartitionClassStorageStatsWrapper} to generate two {@link AggregatedPartitionClassStorageStats}s. First
 * {@link AggregatedPartitionClassStorageStats} is the sum of all {@link HostPartitionClassStorageStatsWrapper}s. The second
 * {@link AggregatedPartitionClassStorageStats} is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostPartitionClassStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedPartitionClassStorageStats}.
 * @throws IOException
 */
Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregateHostPartitionClassStorageStatsWrappers(Map<String, HostPartitionClassStorageStatsWrapper> statsWrappers) throws IOException {
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> combinedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> selectedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostPartitionClassStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostPartitionClassStorageStatsWrapper hostPartitionClassStorageStatsWrapper = statsWrapperEntry.getValue();
        HostPartitionClassStorageStats hostPartitionClassStorageStats = hostPartitionClassStorageStatsWrapper.getStats();
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy1 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy2 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        combineRawHostPartitionClassStorageStatsMap(combinedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy1.getStorageStats());
        selectRawHostPartitionClassStorageStatsMap(selectedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostPartitionClassStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(combinedHostPartitionClassStorageStatsMap));
        logger.trace("Selected raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(selectedHostPartitionClassStorageStatsMap));
    }
    AggregatedPartitionClassStorageStats combinedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(combinedHostPartitionClassStorageStatsMap));
    AggregatedPartitionClassStorageStats selectedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(selectedHostPartitionClassStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}
Also used : HostPartitionClassStorageStatsWrapper(com.github.ambry.server.HostPartitionClassStorageStatsWrapper) HashMap(java.util.HashMap) HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) AggregatedPartitionClassStorageStats(com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats) HashMap(java.util.HashMap) Map(java.util.Map) Pair(com.github.ambry.utils.Pair)

Example 5 with HostPartitionClassStorageStats

use of com.github.ambry.server.storagestats.HostPartitionClassStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToHostPartitionClassStorageStats.

/**
 * Convert a {@link StatsSnapshot} to a {@link HostPartitionClassStorageStats}. We assume the given {@link StatsSnapshot}
 * follows the proper format so we can construct an {@link HostPartitionClassStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link HostPartitionClassStorageStats}.
 */
public static HostPartitionClassStorageStats convertStatsSnapshotToHostPartitionClassStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    Map<String, StatsSnapshot> partitionClassSubMap = Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> partitionClassSubMapEntry : partitionClassSubMap.entrySet()) {
        String partitionClassName = partitionClassSubMapEntry.getKey();
        Map<String, StatsSnapshot> partitionSubMap = Optional.ofNullable(partitionClassSubMapEntry.getValue().getSubMap()).orElseGet(HashMap::new);
        for (Map.Entry<String, StatsSnapshot> partitionSubMapEntry : partitionSubMap.entrySet()) {
            long partitionId = Utils.partitionIdFromStatsPartitionKey(partitionSubMapEntry.getKey());
            Map<String, StatsSnapshot> accountContainerSubMap = Optional.ofNullable(partitionSubMapEntry.getValue().getSubMap()).orElseGet(HashMap::new);
            for (Map.Entry<String, StatsSnapshot> accountContainerSubMapEntry : accountContainerSubMap.entrySet()) {
                short[] accountContainerIds = Utils.accountContainerIdFromPartitionClassStatsKey(accountContainerSubMapEntry.getKey());
                short accountId = accountContainerIds[0];
                short containerId = accountContainerIds[1];
                long logicalStorageUsage = accountContainerSubMapEntry.getValue().getValue();
                hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
            }
        }
    }
    return hostPartitionClassStorageStats;
}
Also used : ContainerStorageStats(com.github.ambry.server.storagestats.ContainerStorageStats) HashMap(java.util.HashMap) HostPartitionClassStorageStats(com.github.ambry.server.storagestats.HostPartitionClassStorageStats) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

HostPartitionClassStorageStats (com.github.ambry.server.storagestats.HostPartitionClassStorageStats)7 HashMap (java.util.HashMap)7 Map (java.util.Map)7 HostPartitionClassStorageStatsWrapper (com.github.ambry.server.HostPartitionClassStorageStatsWrapper)6 ContainerStorageStats (com.github.ambry.server.storagestats.ContainerStorageStats)6 StatsHeader (com.github.ambry.server.StatsHeader)5 AggregatedPartitionClassStorageStats (com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats)5 HostAccountStorageStatsWrapper (com.github.ambry.server.HostAccountStorageStatsWrapper)4 AggregatedAccountStorageStats (com.github.ambry.server.storagestats.AggregatedAccountStorageStats)4 HostAccountStorageStats (com.github.ambry.server.storagestats.HostAccountStorageStats)4 Pair (com.github.ambry.utils.Pair)4 Collections (java.util.Collections)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 StorageStatsUtilTest (com.github.ambry.server.StorageStatsUtilTest)3 Assert (org.junit.Assert)3 Test (org.junit.Test)3 MetricRegistry (com.codahale.metrics.MetricRegistry)2 TypeReference (com.fasterxml.jackson.core.type.TypeReference)2 AccountStatsMySqlConfig (com.github.ambry.config.AccountStatsMySqlConfig)2 Utils (com.github.ambry.utils.Utils)2