Examples with AggregatedAccountStorageStats com.github.ambry.server.storagestats.AggregatedAccountStorageStats used on opensource projects

Example 11 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithEarlyNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} with one node having earlier timestamp
 * but larger storage stats data. Larger data should be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithEarlyNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10L, 2, 10));
    // Use a much larger maximum number when creating early storage stats so the values would be larger here.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> earlyStorageStatsMap = new HashMap<>();
    earlyStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(RELEVANT_PERIOD_IN_MINUTES / 2), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper earlyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(earlyStorageStatsMap));
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", earlyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(earlyStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 12 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStats() throws Exception {
    int nodeCount = 3;
    // Partition id to account id to container id to container storage stats.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap = new HashMap<>();
    // Create storage stats for 3 partitions, each has 3, 4 and 5 accounts.
    for (int i = 0; i < 3; i++) {
        storageStatsMap.put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    // So the combined raw data would be three times of the aggregated data (we have 3 nodes).
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostAccountStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostAccountStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostAccountStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForAccountStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 13 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedAccountStorageStats.

/**
 * Test the methods for storing, deleting and fetch aggregated account storage stats.
 * @throws Exception
 */
@Test
public void testAggregatedAccountStorageStats() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    // Compare container usage map
    Map<String, Map<String, Long>> obtainedContainerStorageUsages = mySqlStore.queryAggregatedAccountStats(false);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(aggregatedAccountStorageStats, false), obtainedContainerStorageUsages);
    // Compare StatsSnapshot
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName1);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(aggregatedAccountStorageStats, false), obtainedSnapshot);
    // Compare AggregatedAccountStorageStats
    AggregatedAccountStorageStats obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(aggregatedAccountStorageStats.getStorageStats(), obtainedStats.getStorageStats());
    // Fetching aggregated account stats for clustername2 should result in a null;
    assertEquals(mySqlStore.queryAggregatedAccountStatsByClusterName(clusterName2).getSubMap().size(), 0);
    assertEquals(mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName2).getStorageStats().size(), 0);
    // Change one value and store it to mysql database again
    Map<Short, Map<Short, ContainerStorageStats>> newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    ContainerStorageStats origin = newStorageStatsMap.get((short) 1).get((short) 1);
    newStorageStatsMap.get((short) 1).put((short) 1, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    aggregatedAccountStorageStats = new AggregatedAccountStorageStats(newStorageStatsMap);
    mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStats();
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    // Delete account and container
    newStorageStatsMap = new HashMap<>(aggregatedAccountStorageStats.getStorageStats());
    newStorageStatsMap.remove((short) 1);
    newStorageStatsMap.get((short) 2).remove((short) 1);
    // Now remove all containers for account 1 and container 1 of account 2
    for (short containerId : aggregatedAccountStorageStats.getStorageStats().get((short) 1).keySet()) {
        mySqlStore.deleteAggregatedAccountStatsForContainer((short) 1, containerId);
    }
    mySqlStore.deleteAggregatedAccountStatsForContainer((short) 2, (short) 1);
    obtainedStats = mySqlStore.queryAggregatedAccountStorageStatsByClusterName(clusterName1);
    assertEquals(newStorageStatsMap, obtainedStats.getStorageStats());
    mySqlStore.shutdown();
}

Example 14 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testMonthlyAggregatedStats.

/**
 * Test methods to store, delete and fetch monthly aggregated stats
 * @throws Exception
 */
@Test
public void testMonthlyAggregatedStats() throws Exception {
    String monthValue = "2020-01";
    AggregatedAccountStorageStats currentAggregatedStats = mySqlStore.queryAggregatedAccountStorageStats();
    if (currentAggregatedStats.getStorageStats().size() == 0) {
        AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 10, 10, 10000L, 2, 10));
        mySqlStore.storeAggregatedAccountStorageStats(aggregatedAccountStorageStats);
        currentAggregatedStats = mySqlStore.queryAggregatedAccountStorageStats();
    }
    // fetch the month and it should return emtpy string
    Assert.assertEquals("", mySqlStore.queryRecordedMonth());
    mySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(monthValue);
    Map<String, Map<String, Long>> monthlyContainerStorageUsages = mySqlStore.queryMonthlyAggregatedAccountStats(false);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(currentAggregatedStats, false), monthlyContainerStorageUsages);
    String obtainedMonthValue = mySqlStore.queryRecordedMonth();
    assertTrue(obtainedMonthValue.equals(monthValue));
    // Change the value and store it back to mysql database
    monthValue = "2020-02";
    currentAggregatedStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedAccountStorageStats(currentAggregatedStats);
    mySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(monthValue);
    monthlyContainerStorageUsages = mySqlStore.queryMonthlyAggregatedAccountStats(false);
    assertEquals(StorageStatsUtil.convertAggregatedAccountStorageStatsToMap(currentAggregatedStats, false), monthlyContainerStorageUsages);
    obtainedMonthValue = mySqlStore.queryRecordedMonth();
    assertTrue(obtainedMonthValue.equals(monthValue));
    // Delete the snapshots
    mySqlStore.deleteSnapshotOfAggregatedAccountStats();
    assertTrue(mySqlStore.queryMonthlyAggregatedAccountStats(false).isEmpty());
}

Example 15 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class GetStatsReportHandlerTest method handleGoodCaseTest.

@Test
public void handleGoodCaseTest() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 1, 10, 10, 1000L, 2, 100));
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedAccountStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedAccountStorageStatsByClusterName(anyString());
    RestRequest restRequest = createRestRequest(CLUSTER_NAME, StatsReportType.ACCOUNT_REPORT.name());
    RestResponseChannel restResponseChannel = new MockRestResponseChannel();
    ReadableStreamChannel channel = sendRequestGetResponse(restRequest, restResponseChannel);
    assertNotNull("There should be a response", channel);
    assertNotNull("Date has not been set", restResponseChannel.getHeader(RestUtils.Headers.DATE));
    assertEquals("Content-type is not as expected", RestUtils.JSON_CONTENT_TYPE, restResponseChannel.getHeader(RestUtils.Headers.CONTENT_TYPE));
    assertEquals("Content-length is not as expected", channel.getSize(), Integer.parseInt((String) restResponseChannel.getHeader(RestUtils.Headers.CONTENT_LENGTH)));
    assertEquals("Storage stats mismatch", aggregatedAccountStorageStats.getStorageStats(), mapper.readValue(RestTestUtils.getResponseBody(channel), AggregatedAccountStorageStats.class).getStorageStats());
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(new String[] { "default", "newClass" }, (short) 1, 10, 10, 1000L, 2, 100));
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedPartitionClassStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedPartitionClassStorageStatsByClusterName(anyString());
    restRequest = createRestRequest(CLUSTER_NAME, StatsReportType.PARTITION_CLASS_REPORT.name());
    restResponseChannel = new MockRestResponseChannel();
    channel = sendRequestGetResponse(restRequest, restResponseChannel);
    assertNotNull("There should be a response", channel);
    assertNotNull("Date has not been set", restResponseChannel.getHeader(RestUtils.Headers.DATE));
    assertEquals("Content-type is not as expected", RestUtils.JSON_CONTENT_TYPE, restResponseChannel.getHeader(RestUtils.Headers.CONTENT_TYPE));
    assertEquals("Content-length is not as expected", channel.getSize(), Integer.parseInt((String) restResponseChannel.getHeader(RestUtils.Headers.CONTENT_LENGTH)));
    assertEquals("Storage stats mismatch", aggregatedPartitionClassStorageStats.getStorageStats(), mapper.readValue(RestTestUtils.getResponseBody(channel), AggregatedPartitionClassStorageStats.class).getStorageStats());
}