Examples with StatsSnapshot com.github.ambry.server.StatsSnapshot used on opensource projects

Example 31 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedAccountStorageStats.

/**
 * Test the methods for storing, deleting and fetch aggregated account storage stats.
 * @throws Exception
 */
@Test
public void testAggregatedAccountStorageStats() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    // Compare container usage map
    Map<String, Map<String, Long>> obtainedContainerStorageUsages = mySqlStore.queryAggregatedAccountStats(false);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(aggregatedAccountStorageStats, false), obtainedContainerStorageUsages);
    // Compare StatsSnapshot
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName1);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(aggregatedAccountStorageStats, false), obtainedSnapshot);
    // Compare AggregatedAccountStorageStats
    AggregatedAccountStorageStats obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    // Fetching aggregated account stats for clustername2 should result in a null;
    assertEquals(mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName2).getSubMap().size(), 0);
    assertEquals(mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName2).getStorageStats().size(), 0);
    // Change one value and store it to mysql database again
    Map<Short, Map<Short, ContainerStorageStats>> newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    ContainerStorageStats origin = newStorageStatsMap.get((short) 1).get((short) 1);
    newStorageStatsMap.get((short) 1).put((short) 1, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    aggregatedAccountStorageStats = new AggregatedAccountStorageStats(newStorageStatsMap);
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    // Delete account and container
    newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    newStorageStatsMap.remove((short) 1);
    newStorageStatsMap.get((short) 2).remove((short) 1);
    // Now remove all containers for account 1 and container 1 of account 2
    for (short containerId : aggregatedAccountStorageStats.getStorageStats().get((short) 1).keySet()) {
        mySqlStore.deleteAggregatedAccountStatsForContainer((short) 1, containerId);
    }
    mySqlStore.deleteAggregatedAccountStatsForContainer((short) 2, (short) 1);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    mySqlStore.shutdown();
}

Example 32 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedAccountStats.

/**
 * Test the methods for storing, deleting and fetch aggregated account stats.
 * @throws Exception
 */
@Test
public void testAggregatedAccountStats() throws Exception {
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    mySqlStore.storeAggregatedAccountStats(snapshot);
    Map<String, Map<String, Long>> obtainedContainerStorageUsages = mySqlStore.queryAggregatedAccountStats(false);
    assertEquals(containerStorageUsages, obtainedContainerStorageUsages);
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName1);
    assertEquals(snapshot, obtainedSnapshot);
    // Fetching aggregated account stats for clustername2 should result in empty stats
    assertEquals(mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName2).getSubMap().size(), 0);
    // Change one value and store it to mysql database again
    StatsSnapshot newSnapshot = new StatsSnapshot(snapshot);
    newSnapshot.getSubMap().get(Utils.statsAccountKey((short) 1)).getSubMap().get(Utils.statsContainerKey((short) 1)).setValue(1);
    newSnapshot.updateValue();
    containerStorageUsages.get("1").put("1", 1L);
    mySqlStore.storeAggregatedAccountStats(newSnapshot);
    obtainedContainerStorageUsages = mySqlStore.queryAggregatedAccountStats(false);
    assertEquals(containerStorageUsages, obtainedContainerStorageUsages);
    // Delete account and container
    newSnapshot = new StatsSnapshot(newSnapshot);
    newSnapshot.getSubMap().remove(Utils.statsAccountKey((short) 1));
    newSnapshot.getSubMap().get(Utils.statsAccountKey((short) 2)).getSubMap().remove(Utils.statsContainerKey((short) 1));
    newSnapshot.updateValue();
    // Now remove all containers for account 1 and container 1 of account 2
    for (String containerId : containerStorageUsages.get(String.valueOf(1)).keySet()) {
        mySqlStore.deleteAggregatedAccountStatsForContainer((short) 1, Short.valueOf(containerId));
    }
    mySqlStore.deleteAggregatedAccountStatsForContainer((short) 2, (short) 1);
    obtainedSnapshot = mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName1);
    assertEquals(newSnapshot, obtainedSnapshot);
    mySqlStore.shutdown();
}

Example 33 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testRefresherUpdateAndListener.

/**
 * Test when updating container total usage.
 * @throws Exception
 */
@Test
public void testRefresherUpdateAndListener() throws Exception {
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    // Set polling interval to 2 seconds
    properties.setProperty(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS, "2000");
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
    AtomicReference<Map<String, Map<String, Long>>> containerUsageRef = new AtomicReference<>(null);
    AtomicReference<CountDownLatch> latchRef = new AtomicReference<>(null);
    refresher.registerListener(containerStorageUsage -> {
        containerUsageRef.set(containerStorageUsage);
        latchRef.get().countDown();
    });
    // Keep storage usage unchanged, listener should get an all-zero map
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch1 = new CountDownLatch(1);
    latchRef.set(latch1);
    latch1.await(10, TimeUnit.SECONDS);
    assertContainerUsageMapAllZero(containerUsageRef.get());
    // Change some usage, listener should get
    containerStorageUsages.get("1").compute("1", (k, v) -> v + 1L);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch2 = new CountDownLatch(1);
    latchRef.set(latch2);
    latch2.await(10, TimeUnit.SECONDS);
    assertEquals((long) containerUsageRef.get().get("1").get("1"), 1L);
    Map<String, Map<String, Long>> clone = cloneMap(containerUsageRef.get());
    clone.get("1").put("1", 0L);
    assertContainerUsageMapAllZero(clone);
}

Example 34 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class TestUtils method makePartitionClassSnasphotFromContainerStorageMap.

/**
 * Generating a partition class snapshot from the given container storage map.
 * @param containerStorageMap The container storage map.
 * @return A partition class {@link StatsSnapshot}.
 */
public static StatsSnapshot makePartitionClassSnasphotFromContainerStorageMap(Map<String, Map<String, Map<String, Long>>> containerStorageMap) {
    Map<String, StatsSnapshot> partitionClassSnapshots = new HashMap<>();
    long sumOfPartitionClassUsage = 0;
    for (Map.Entry<String, Map<String, Map<String, Long>>> partitionClassUsageEntry : containerStorageMap.entrySet()) {
        String partitionClassName = partitionClassUsageEntry.getKey();
        Map<String, StatsSnapshot> accountContainerSnapshots = new HashMap<>();
        long sumOfAccountContainerUsage = 0;
        Map<String, Map<String, Long>> accountContainerStorageUsage = partitionClassUsageEntry.getValue();
        for (Map.Entry<String, Map<String, Long>> accountContainerEntry : accountContainerStorageUsage.entrySet()) {
            String accountId = accountContainerEntry.getKey();
            for (Map.Entry<String, Long> containerEntry : accountContainerEntry.getValue().entrySet()) {
                String containerId = containerEntry.getKey();
                accountContainerSnapshots.put(Utils.partitionClassStatsAccountContainerKey(Short.valueOf(accountId), Short.valueOf(containerId)), new StatsSnapshot(containerEntry.getValue(), null));
                sumOfAccountContainerUsage += containerEntry.getValue();
            }
        }
        partitionClassSnapshots.put(partitionClassName, new StatsSnapshot(sumOfAccountContainerUsage, accountContainerSnapshots));
        sumOfPartitionClassUsage += sumOfAccountContainerUsage;
    }
    return new StatsSnapshot(sumOfPartitionClassUsage, partitionClassSnapshots);
}

Example 35 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class TestUtils method makeAggregatedPartitionClassStats.

/**
 * Generating an aggregated partition class stats. {@code partitionClassNames} provides a list of partition class names
 * and for each partition class name, {@code numAccount} * {@code numContainer} container will be created.
 * @param partitionClassNames The list of partition class names.
 * @param numAccount The number of accounts.
 * @param numContainer The number of containers per account.
 * @return An aggregated partition class {@link StatsSnapshot}.
 */
public static StatsSnapshot makeAggregatedPartitionClassStats(String[] partitionClassNames, int numAccount, int numContainer) {
    Random random = new Random();
    long maxValue = 10000;
    StatsSnapshot finalStats = new StatsSnapshot(0L, new HashMap<>());
    for (String className : partitionClassNames) {
        StatsSnapshot classNameStats = new StatsSnapshot(0L, new HashMap<>());
        finalStats.getSubMap().put(className, classNameStats);
        for (int ia = 0; ia < numAccount; ia++) {
            for (int ic = 0; ic < numContainer; ic++) {
                String key = Utils.partitionClassStatsAccountContainerKey((short) ia, (short) ic);
                classNameStats.getSubMap().put(key, new StatsSnapshot(Math.abs(random.nextLong() % maxValue), null));
            }
        }
        long classNameValue = classNameStats.getSubMap().values().stream().mapToLong(StatsSnapshot::getValue).sum();
        classNameStats.setValue(classNameValue);
    }
    finalStats.setValue(finalStats.getSubMap().values().stream().mapToLong(StatsSnapshot::getValue).sum());
    return finalStats;
}