Examples with HostAccountStorageStats com.github.ambry.server.storagestats.HostAccountStorageStats used on opensource projects

Example 1 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class StorageStatsTest method testHostAccountStorageStats.

/**
 * Test methods in {@link HostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testHostAccountStorageStats() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = new HashMap<>();
    long partitionId = 10000L;
    short accountId = 0;
    short containerId = 100;
    int numberOfPartitions = 2;
    int numberOfAccounts = 2;
    int numberOfContainers = 2;
    for (int i = 0; i < numberOfPartitions; i++) {
        partitionId++;
        if (!storageStats.containsKey(partitionId)) {
            storageStats.put(partitionId, new HashMap<>());
        }
        for (int j = 0; j < numberOfAccounts; j++) {
            accountId++;
            if (!storageStats.get(partitionId).containsKey(accountId)) {
                storageStats.get(partitionId).put(accountId, new HashMap<>());
            }
            for (int k = 0; k < numberOfContainers; k++) {
                containerId++;
                if (!storageStats.get(partitionId).get(accountId).containsKey(containerId)) {
                    storageStats.get(partitionId).get(accountId).put(containerId, generateRandomContainerStorageStats(containerId));
                }
            }
        }
    }
    HostAccountStorageStats host1 = new HostAccountStorageStats(storageStats);
    HostAccountStorageStats host2 = new HostAccountStorageStats();
    for (Map.Entry<Long, Map<Short, Map<Short, ContainerStorageStats>>> partitionEntry : storageStats.entrySet()) {
        long pid = partitionEntry.getKey();
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> accountEntry : partitionEntry.getValue().entrySet()) {
            short aid = accountEntry.getKey();
            for (Map.Entry<Short, ContainerStorageStats> containerEntry : accountEntry.getValue().entrySet()) {
                host2.addContainerStorageStats(pid, aid, containerEntry.getValue());
            }
        }
    }
    Assert.assertEquals(host1.getStorageStats(), host2.getStorageStats());
    // Serialize the host account storage stats
    String serialized = objectMapper.writeValueAsString(host1);
    HostAccountStorageStats deserialized = objectMapper.readValue(serialized, HostAccountStorageStats.class);
    Assert.assertEquals(host1.getStorageStats(), deserialized.getStorageStats());
}

Example 2 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToHostAccountStorageStats.

/**
 * Convert a {@link StatsSnapshot} to {@link HostAccountStorageStats}. We assume the given {@link StatsSnapshot} follows
 * the proper format so we can construct a {@link HostAccountStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link HostAccountStorageStats}.
 */
public static HostAccountStorageStats convertStatsSnapshotToHostAccountStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats();
    Map<String, StatsSnapshot> partitionMap = // The map whose key is the partition id
    Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> partitionEntry : partitionMap.entrySet()) {
        long partitionId = Utils.partitionIdFromStatsPartitionKey(partitionEntry.getKey());
        Map<String, StatsSnapshot> accountMap = Optional.ofNullable(partitionEntry.getValue().getSubMap()).orElseGet(// The map whose key is the account id
        HashMap::new);
        for (Map.Entry<String, StatsSnapshot> accountEntry : accountMap.entrySet()) {
            short accountId = Utils.accountIdFromStatsAccountKey(accountEntry.getKey());
            Map<String, StatsSnapshot> containerMap = Optional.ofNullable(accountEntry.getValue().getSubMap()).orElseGet(HashMap::new);
            for (Map.Entry<String, StatsSnapshot> containerEntry : containerMap.entrySet()) {
                short containerId = Utils.containerIdFromStatsContainerKey(containerEntry.getKey());
                long logicalStorageUsage = containerEntry.getValue().getValue();
                hostAccountStorageStats.addContainerStorageStats(partitionId, accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
            }
        }
    }
    return hostAccountStorageStats;
}

Example 3 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithDifferentNumberOfStores.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers}, but with different numbers of partitions
 * from different nodes.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithDifferentNumberOfStores() throws Exception {
    // This storage stats has only 2 partitions, partition 0 and partition 1. Each partition has only one account and one container.
    String statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':10, 'physicalStorageUsage':20, 'numberOfBlobs':10}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':20, 'physicalStorageUsage':40, 'numberOfBlobs':20}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap1 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // This storage stats has only 3 partitions, partition 0, partition 1 and partition 2. Each partition has only one account and one container.
    statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':30, 'physicalStorageUsage':60, 'numberOfBlobs':30}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':40, 'physicalStorageUsage':80, 'numberOfBlobs':40}}}, '2': {'0': {'0': {'containerId':0, 'logicalStorageUsage':50, 'physicalStorageUsage':100, 'numberOfBlobs':50}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap2 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // Raw combined storage stats should only have one account and one container, and its storage stats is the sum of storage stats from all nodes.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':150, 'physicalStorageUsage':300, 'numberOfBlobs':150}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedRaw = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // Aggregated storage stats should only have one account and one container, and its storage stats is the sum of storage stats from second node,
    // since second node's physical storage usage is larger than first node at every partition.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':120, 'physicalStorageUsage':240, 'numberOfBlobs':120}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // StorageStatsMap1 only have 2 store stats,  StorageStatsMap2 has 3
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 2, 2, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats1 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap1));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats2 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap2));
    Map<String, HostAccountStorageStatsWrapper> instanceStatsMap = new HashMap<>();
    instanceStatsMap.put("Instance_1", nodeStats1);
    instanceStatsMap.put("Instance_2", nodeStats2);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceStatsMap);
    Assert.assertEquals(expectedRaw, aggregatedRawAndValidStats.getFirst().getStorageStats());
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 4 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithOutdatedNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} but with one node having outdated
 * storage stats data. Outdated data shouldn't be used when aggregating valid storage usage.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithOutdatedNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> outdatedStorageStatsMap = new HashMap<>();
    outdatedStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper outdatedNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(outdatedStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", outdatedNodeStats);
    instanceToStatsMap.put("Instance_2", emptyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(upToDateStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 5 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStoreMultilpleWrites.

/**
 * Tests to store multiple stats for one hosts and recover stats from database.
 * @throws Exception
 */
@Test
public void testStoreMultilpleWrites() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats1 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats1);
    HostAccountStorageStats hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    ContainerStorageStats origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats2.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).physicalStorageUsage(origin.getPhysicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats3);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats3.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).numberOfBlobs(origin.getNumberOfBlobs() + 1).build());
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats4);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats4.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}