Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 31 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithEarlyNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} with one node having earlier timestamp
 * but larger storage stats data. Larger data should be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithEarlyNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10L, 2, 10));
    // Use a much larger maximum number when creating early storage stats so the values would be larger here.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> earlyStorageStatsMap = new HashMap<>();
    earlyStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(RELEVANT_PERIOD_IN_MINUTES / 2), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper earlyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(earlyStorageStatsMap));
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", earlyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(earlyStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 32 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostPartitionClassStorageStatsWithOutdatedNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostPartitionClassStorageStatsWrappers} but with one node having outdated
 * storage stats data. Outdated data shouldn't be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostPartitionClassStorageStatsWithOutdatedNode() throws Exception {
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.computeIfAbsent("default", k -> new HashMap<>()).put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> outdatedStorageStatsMap = new HashMap<>();
    outdatedStorageStatsMap.computeIfAbsent("default", k -> new HashMap<>()).put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 1, 1, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper upToDateNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper outdatedNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(outdatedStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 0, 0, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper emptyNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats());
    Map<String, HostPartitionClassStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", outdatedNodeStats);
    instanceToStatsMap.put("Instance_2", emptyNodeStats);
    Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(instanceToStatsMap);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> expectedValid = clusterAggregator.aggregateHostPartitionClassStorageStats(upToDateStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 33 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStats() throws Exception {
    int nodeCount = 3;
    // Partition id to account id to container id to container storage stats.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap = new HashMap<>();
    // Create storage stats for 3 partitions, each has 3, 4 and 5 accounts.
    for (int i = 0; i < 3; i++) {
        storageStatsMap.put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    // So the combined raw data would be three times of the aggregated data (we have 3 nodes).
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostAccountStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostAccountStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostAccountStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForAccountStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 34 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedPartitionClassStorageStats.

@Test
public void testAggregatedPartitionClassStorageStats() throws Exception {
    testHostPartitionClassStorageStats();
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    // Now we should have partition class names and partition ids in database
    // Construct an aggregated partition class report
    AggregatedPartitionClassStorageStats aggregatedStats = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(partitionNameAndIds.keySet().toArray(new String[0]), (short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedPartitionClassStorageStats(aggregatedStats);
    partitionNameAndIds = mySqlStore3.queryPartitionNameAndIds();
    AggregatedPartitionClassStorageStats aggregatedStats3 = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(partitionNameAndIds.keySet().toArray(new String[0]), (short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore3.storeAggregatedPartitionClassStorageStats(aggregatedStats3);
    AggregatedPartitionClassStorageStats obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(aggregatedStats.getStorageStats(), obtained.getStorageStats());
    assertEquals(mySqlStore.queryAggregatedPartitionClassStorageStatsByClusterName("random-cluster").getStorageStats().size(), 0);
    AggregatedPartitionClassStorageStats obtained3 = mySqlStore3.queryAggregatedPartitionClassStorageStats();
    assertEquals(aggregatedStats3.getStorageStats(), obtained3.getStorageStats());
    // Fetch StatsSnapshot
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedPartitionClassStats();
    assertEquals(StorageStatsUtil.convertAggregatedPartitionClassStorageStatsToStatsSnapshot(obtained, false), obtainedSnapshot);
    // Change one value and store it to mysql database again
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(aggregatedStats.getStorageStats());
    ContainerStorageStats origin = newStorageStatsMap.get("default").get((short) 1).get((short) 1);
    newStorageStatsMap.get("default").get((short) 1).put((short) 1, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    mySqlStore.storeAggregatedPartitionClassStorageStats(new AggregatedPartitionClassStorageStats(newStorageStatsMap));
    obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(newStorageStatsMap, obtained.getStorageStats());
    // Delete some account and container
    short accountId = (short) 1;
    short containerId = (short) 1;
    for (String partitionClassName : partitionNameAndIds.keySet()) {
        mySqlStore.deleteAggregatedPartitionClassStatsForAccountContainer(partitionClassName, accountId, containerId);
        newStorageStatsMap.get(partitionClassName).get(accountId).remove(containerId);
    }
    obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(newStorageStatsMap, obtained.getStorageStats());
    mySqlStore3.shutdown();
}

Example 35 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedAccountStorageStats.

/**
 * Test the methods for storing, deleting and fetch aggregated account storage stats.
 * @throws Exception
 */
@Test
public void testAggregatedAccountStorageStats() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    // Compare container usage map
    Map<String, Map<String, Long>> obtainedContainerStorageUsages = mySqlStore.queryAggregatedAccountStats(false);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(aggregatedAccountStorageStats, false), obtainedContainerStorageUsages);
    // Compare StatsSnapshot
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName1);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(aggregatedAccountStorageStats, false), obtainedSnapshot);
    // Compare AggregatedAccountStorageStats
    AggregatedAccountStorageStats obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    // Fetching aggregated account stats for clustername2 should result in a null;
    assertEquals(mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName2).getSubMap().size(), 0);
    assertEquals(mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName2).getStorageStats().size(), 0);
    // Change one value and store it to mysql database again
    Map<Short, Map<Short, ContainerStorageStats>> newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    ContainerStorageStats origin = newStorageStatsMap.get((short) 1).get((short) 1);
    newStorageStatsMap.get((short) 1).put((short) 1, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    aggregatedAccountStorageStats = new AggregatedAccountStorageStats(newStorageStatsMap);
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    // Delete account and container
    newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    newStorageStatsMap.remove((short) 1);
    newStorageStatsMap.get((short) 2).remove((short) 1);
    // Now remove all containers for account 1 and container 1 of account 2
    for (short containerId : aggregatedAccountStorageStats.getStorageStats().get((short) 1).keySet()) {
        mySqlStore.deleteAggregatedAccountStatsForContainer((short) 1, containerId);
    }
    mySqlStore.deleteAggregatedAccountStatsForContainer((short) 2, (short) 1);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    mySqlStore.shutdown();
}