Examples with AggregatedAccountStorageStats com.github.ambry.server.storagestats.AggregatedAccountStorageStats used on opensource projects

Example 1 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithDifferentNumberOfStores.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers}, but with different numbers of partitions
 * from different nodes.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithDifferentNumberOfStores() throws Exception {
    // This storage stats has only 2 partitions, partition 0 and partition 1. Each partition has only one account and one container.
    String statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':10, 'physicalStorageUsage':20, 'numberOfBlobs':10}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':20, 'physicalStorageUsage':40, 'numberOfBlobs':20}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap1 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // This storage stats has only 3 partitions, partition 0, partition 1 and partition 2. Each partition has only one account and one container.
    statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':30, 'physicalStorageUsage':60, 'numberOfBlobs':30}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':40, 'physicalStorageUsage':80, 'numberOfBlobs':40}}}, '2': {'0': {'0': {'containerId':0, 'logicalStorageUsage':50, 'physicalStorageUsage':100, 'numberOfBlobs':50}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap2 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // Raw combined storage stats should only have one account and one container, and its storage stats is the sum of storage stats from all nodes.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':150, 'physicalStorageUsage':300, 'numberOfBlobs':150}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedRaw = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // Aggregated storage stats should only have one account and one container, and its storage stats is the sum of storage stats from second node,
    // since second node's physical storage usage is larger than first node at every partition.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':120, 'physicalStorageUsage':240, 'numberOfBlobs':120}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // StorageStatsMap1 only have 2 store stats,  StorageStatsMap2 has 3
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 2, 2, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats1 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap1));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats2 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap2));
    Map<String, HostAccountStorageStatsWrapper> instanceStatsMap = new HashMap<>();
    instanceStatsMap.put("Instance_1", nodeStats1);
    instanceStatsMap.put("Instance_2", nodeStats2);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceStatsMap);
    Assert.assertEquals(expectedRaw, aggregatedRawAndValidStats.getFirst().getStorageStats());
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 2 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithOutdatedNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} but with one node having outdated
 * storage stats data. Outdated data shouldn't be used when aggregating valid storage usage.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithOutdatedNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> outdatedStorageStatsMap = new HashMap<>();
    outdatedStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper outdatedNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(outdatedStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", outdatedNodeStats);
    instanceToStatsMap.put("Instance_2", emptyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(upToDateStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 3 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class FrontendTestUrlSigningServiceFactory method getStatsReportTest.

/**
 * Tests the handling of {@link Operations#STATS_REPORT} get requests.
 * @throws Exception
 */
@Test
public void getStatsReportTest() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 1, 10, 10, 1000L, 2, 100));
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(new String[] { "default", "newClass" }, (short) 1, 10, 10, 1000L, 2, 100));
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedAccountStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedAccountStorageStatsByClusterName(anyString());
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedPartitionClassStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedPartitionClassStorageStatsByClusterName(anyString());
    ObjectMapper mapper = new ObjectMapper();
    // construct a request to get account stats
    JSONObject headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, CLUSTER_NAME);
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.ACCOUNT_REPORT.name());
    RestRequest request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    MockRestResponseChannel restResponseChannel = new MockRestResponseChannel();
    doOperation(request, restResponseChannel);
    assertEquals("Storage stats mismatch", aggregatedAccountStorageStats.getStorageStats(), mapper.readValue(restResponseChannel.getResponseBody(), AggregatedAccountStorageStats.class).getStorageStats());
    // construct a request to get partition class stats
    headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, CLUSTER_NAME);
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.PARTITION_CLASS_REPORT.name());
    request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    restResponseChannel = new MockRestResponseChannel();
    doOperation(request, restResponseChannel);
    assertEquals("Storage stats mismatch", aggregatedPartitionClassStorageStats.getStorageStats(), mapper.readValue(restResponseChannel.getResponseBody(), AggregatedPartitionClassStorageStats.class).getStorageStats());
    // test clustername not found case to ensure that it goes through the exception path
    headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, "WRONG_CLUSTER");
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.ACCOUNT_REPORT.name());
    request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    try {
        doOperation(request, new MockRestResponseChannel());
        fail("Operation should have failed");
    } catch (RestServiceException e) {
        assertEquals("ErrorCode not as expected", RestServiceErrorCode.NotFound, e.getErrorCode());
    }
}

Example 4 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method queryMonthlyAggregatedAccountStorageStats.

@Override
public AggregatedAccountStorageStats queryMonthlyAggregatedAccountStorageStats() throws Exception {
    long startTimeMs = System.currentTimeMillis();
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(null);
    aggregatedAccountReportsDao.queryMonthlyContainerUsageForCluster(clusterName, (accountId, containerStats) -> {
        aggregatedAccountStorageStats.addContainerStorageStats(accountId, containerStats);
    });
    storeMetrics.queryMonthlyAggregatedStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return aggregatedAccountStorageStats;
}

Example 5 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class StorageStatsUtilTest method testAggregatedAccountStorageStatsConverter.

/**
 * Test case for {@link StorageStatsUtil#convertAggregatedAccountStorageStatsToStatsSnapshot}.
 */
@Test
public void testAggregatedAccountStorageStatsConverter() {
    Map<Short, Map<Short, ContainerStorageStats>> storageStats = generateRandomAggregatedAccountStorageStats((short) 1000, 10, 10, 10000L, 2, 10);
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(storageStats);
    StatsSnapshot expected = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(aggregatedAccountStorageStats, false));
    StatsSnapshot snapshot = StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(aggregatedAccountStorageStats, false);
    Assert.assertEquals(expected, snapshot);
    expected = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(aggregatedAccountStorageStats, true));
    snapshot = StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(aggregatedAccountStorageStats, true);
    Assert.assertEquals(expected, snapshot);
}