Examples with HostAccountStorageStatsWrapper com.github.ambry.server.HostAccountStorageStatsWrapper used on opensource projects

Example 11 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithEarlyNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} with one node having earlier timestamp
 * but larger storage stats data. Larger data should be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithEarlyNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10L, 2, 10));
    // Use a much larger maximum number when creating early storage stats so the values would be larger here.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> earlyStorageStatsMap = new HashMap<>();
    earlyStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(RELEVANT_PERIOD_IN_MINUTES / 2), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper earlyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(earlyStorageStatsMap));
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", earlyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(earlyStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 12 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStats() throws Exception {
    int nodeCount = 3;
    // Partition id to account id to container id to container storage stats.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap = new HashMap<>();
    // Create storage stats for 3 partitions, each has 3, 4 and 5 accounts.
    for (int i = 0; i < 3; i++) {
        storageStatsMap.put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    // So the combined raw data would be three times of the aggregated data (we have 3 nodes).
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostAccountStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostAccountStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostAccountStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForAccountStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 13 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method generateHostAccountStorageStatsWrapper.

private static HostAccountStorageStatsWrapper generateHostAccountStorageStatsWrapper(int numPartitions, int numAccounts, int numContainersPerAccount, StatsReportType reportType) {
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats(StorageStatsUtilTest.generateRandomHostAccountStorageStats(numPartitions, numAccounts, numContainersPerAccount, 100000L, 2, 10));
    StatsHeader statsHeader = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 1000, numPartitions, numPartitions, Collections.emptyList());
    return new HostAccountStorageStatsWrapper(statsHeader, hostAccountStorageStats);
}

Example 14 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testMultiStoreStats.

/**
 * Tests to store multiple stats for multiple hosts and recover stats from database.
 * @throws Exception
 */
@Test
public void testMultiStoreStats() throws Exception {
    AccountStatsMySqlStore mySqlStore1 = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    AccountStatsMySqlStore mySqlStore2 = createAccountStatsMySqlStore(clusterName1, hostname2, port2);
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    // Generating StatsWrappers, store StatsWrappers and retrieve StatsWrappers
    StatsWrapper stats1 = generateStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    StatsWrapper stats2 = generateStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    StatsWrapper stats3 = generateStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore1.storeAccountStats(stats1);
    mySqlStore2.storeAccountStats(stats2);
    mySqlStore3.storeAccountStats(stats3);
    assertTableSize(mySqlStore1, 3 * 10 * 10);
    StatsWrapper obtainedStats1 = mySqlStore1.queryAccountStatsByHost(hostname1, port1);
    StatsWrapper obtainedStats2 = mySqlStore2.queryAccountStatsByHost(hostname2, port2);
    StatsWrapper obtainedStats3 = mySqlStore3.queryAccountStatsByHost(hostname3, port3);
    assertTwoStatsSnapshots(obtainedStats1.getSnapshot(), stats1.getSnapshot());
    assertTwoStatsSnapshots(obtainedStats2.getSnapshot(), stats2.getSnapshot());
    assertTwoStatsSnapshots(obtainedStats3.getSnapshot(), stats3.getSnapshot());
    // Generating HostAccountStorageStatsWrappers, store and retrieve them
    HostAccountStorageStatsWrapper hostStats1 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    HostAccountStorageStatsWrapper hostStats2 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    HostAccountStorageStatsWrapper hostStats3 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore1.storeHostAccountStorageStats(hostStats1);
    mySqlStore2.storeHostAccountStorageStats(hostStats2);
    mySqlStore3.storeHostAccountStorageStats(hostStats3);
    HostAccountStorageStatsWrapper obtainedHostStats1 = mySqlStore1.queryHostAccountStorageStatsByHost(hostname1, port1);
    HostAccountStorageStatsWrapper obtainedHostStats2 = mySqlStore2.queryHostAccountStorageStatsByHost(hostname2, port2);
    HostAccountStorageStatsWrapper obtainedHostStats3 = mySqlStore3.queryHostAccountStorageStatsByHost(hostname3, port3);
    assertEquals(hostStats1.getStats().getStorageStats(), obtainedHostStats1.getStats().getStorageStats());
    assertEquals(hostStats2.getStats().getStorageStats(), obtainedHostStats2.getStats().getStorageStats());
    assertEquals(hostStats3.getStats().getStorageStats(), obtainedHostStats3.getStats().getStorageStats());
    // Retrieve StatWrappers
    obtainedStats1 = mySqlStore1.queryAccountStatsByHost(hostname1, port1);
    StatsSnapshot converted = StorageStatsUtil.convertHostAccountStorageStatsToStatsSnapshot(hostStats1.getStats(), false);
    assertTwoStatsSnapshots(converted, obtainedStats1.getSnapshot());
    mySqlStore1.shutdown();
    mySqlStore2.shutdown();
    mySqlStore3.shutdown();
}

Example 15 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromMysqlDb.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromMysqlDb() throws Exception {
    // write a new stats into database.
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(1, 1, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
    // initialized the mySqlStore and write a new stats with the same partition.
    mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    assertTrue(mySqlStore.getPreviousHostAccountStorageStatsWrapper().getStats().getStorageStats().containsKey((long) 0));
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(0, 0, 0, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats2.getStats().getStorageStats());
    newStorageStats.put((long) 0, // Remove partition 0's storage stats data, this would remove entire partition from database
    new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    // empty stats should remove all the data in the database
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
}