Examples with HostAccountStorageStats com.github.ambry.server.storagestats.HostAccountStorageStats used on opensource projects

Example 11 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostAccountStorageStatsWrappers.

/**
 * Aggregate all {@link HostAccountStorageStatsWrapper} to generate two {@link AggregatedAccountStorageStats}s. First
 * {@link AggregatedAccountStorageStats} is the sum of all {@link HostAccountStorageStatsWrapper}s. The second {@link AggregatedAccountStorageStats}
 * is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostAccountStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedAccountStorageStats}.
 * @throws IOException
 */
Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregateHostAccountStorageStatsWrappers(Map<String, HostAccountStorageStatsWrapper> statsWrappers) throws IOException {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> combinedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> selectedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostAccountStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostAccountStorageStatsWrapper hostAccountStorageStatsWrapper = statsWrapperEntry.getValue();
        HostAccountStorageStats hostAccountStorageStats = hostAccountStorageStatsWrapper.getStats();
        HostAccountStorageStats hostAccountStorageStatsCopy1 = new HostAccountStorageStats(hostAccountStorageStats);
        HostAccountStorageStats hostAccountStorageStatsCopy2 = new HostAccountStorageStats(hostAccountStorageStats);
        combineRawHostAccountStorageStatsMap(combinedHostAccountStorageStatsMap, hostAccountStorageStatsCopy1.getStorageStats());
        selectRawHostAccountStorageStatsMap(selectedHostAccountStorageStatsMap, hostAccountStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostAccountStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostAccountStorageStats {}", mapper.writeValueAsString(combinedHostAccountStorageStatsMap));
        logger.trace("Selected raw HostAccountStorageStats {}", mapper.writeValueAsString(selectedHostAccountStorageStatsMap));
    }
    AggregatedAccountStorageStats combinedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(combinedHostAccountStorageStatsMap));
    AggregatedAccountStorageStats selectedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(selectedHostAccountStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}

Example 12 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithEarlyNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} with one node having earlier timestamp
 * but larger storage stats data. Larger data should be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithEarlyNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10L, 2, 10));
    // Use a much larger maximum number when creating early storage stats so the values would be larger here.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> earlyStorageStatsMap = new HashMap<>();
    earlyStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(RELEVANT_PERIOD_IN_MINUTES / 2), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper earlyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(earlyStorageStatsMap));
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", earlyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(earlyStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 13 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStats() throws Exception {
    int nodeCount = 3;
    // Partition id to account id to container id to container storage stats.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap = new HashMap<>();
    // Create storage stats for 3 partitions, each has 3, 4 and 5 accounts.
    for (int i = 0; i < 3; i++) {
        storageStatsMap.put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    // So the combined raw data would be three times of the aggregated data (we have 3 nodes).
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostAccountStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostAccountStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostAccountStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForAccountStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 14 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method generateHostAccountStorageStatsWrapper.

private static HostAccountStorageStatsWrapper generateHostAccountStorageStatsWrapper(int numPartitions, int numAccounts, int numContainersPerAccount, StatsReportType reportType) {
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats(StorageStatsUtilTest.generateRandomHostAccountStorageStats(numPartitions, numAccounts, numContainersPerAccount, 100000L, 2, 10));
    StatsHeader statsHeader = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 1000, numPartitions, numPartitions, Collections.emptyList());
    return new HostAccountStorageStatsWrapper(statsHeader, hostAccountStorageStats);
}

Example 15 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromMysqlDb.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromMysqlDb() throws Exception {
    // write a new stats into database.
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(1, 1, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
    // initialized the mySqlStore and write a new stats with the same partition.
    mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    assertTrue(mySqlStore.getPreviousHostAccountStorageStatsWrapper().getStats().getStorageStats().containsKey((long) 0));
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(0, 0, 0, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats2.getStats().getStorageStats());
    newStorageStats.put((long) 0, // Remove partition 0's storage stats data, this would remove entire partition from database
    new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    // empty stats should remove all the data in the database
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
}