Examples with AggregatedPartitionClassStorageStats com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats used on opensource projects

Example 1 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToAggregatedPartitionClassStorageStats.

/**
 * Convert a {@link StatsSnapshot} to an {@link AggregatedPartitionClassStorageStats}. We assume the given {@link StatsSnapshot}
 * follows the proper format so we can construct an {@link AggregatedPartitionClassStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link AggregatedPartitionClassStorageStats}.
 */
public static AggregatedPartitionClassStorageStats convertStatsSnapshotToAggregatedPartitionClassStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(null);
    Map<String, StatsSnapshot> partitionClassNameSubMap = Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> partitionClasNameSubMapEntry : partitionClassNameSubMap.entrySet()) {
        String partitionClassName = partitionClasNameSubMapEntry.getKey();
        Map<String, StatsSnapshot> accountContainerSubMap = Optional.ofNullable(partitionClasNameSubMapEntry.getValue().getSubMap()).orElseGet(HashMap::new);
        for (Map.Entry<String, StatsSnapshot> accountContainerSubMapEntry : accountContainerSubMap.entrySet()) {
            short[] accountContainerIds = Utils.accountContainerIdFromPartitionClassStatsKey(accountContainerSubMapEntry.getKey());
            short accountId = accountContainerIds[0];
            short containerId = accountContainerIds[1];
            long logicalStorageUsage = accountContainerSubMapEntry.getValue().getValue();
            aggregatedPartitionClassStorageStats.addContainerStorageStats(partitionClassName, accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
        }
    }
    return aggregatedPartitionClassStorageStats;
}

Example 2 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostPartitionClassStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostPartitionClassStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostPartitionClassStorageStats() throws Exception {
    int nodeCount = 3;
    int numberOfPartitions = 4;
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> storageStatsMap = new HashMap<>();
    String[] partitionClassNames = { "default", "newClass" };
    for (int i = 0; i < numberOfPartitions; i++) {
        String partitionClassName = partitionClassNames[i % partitionClassNames.length];
        storageStatsMap.computeIfAbsent(partitionClassName, k -> new HashMap<>()).put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, numberOfPartitions, numberOfPartitions, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper nodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper emptyStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats());
    Map<String, HostPartitionClassStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostPartitionClassStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostPartitionClassStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(instanceToStatsMap);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostPartitionClassStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForPartitionClassStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 3 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class FrontendTestUrlSigningServiceFactory method getStatsReportTest.

/**
 * Tests the handling of {@link Operations#STATS_REPORT} get requests.
 * @throws Exception
 */
@Test
public void getStatsReportTest() throws Exception {
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 1, 10, 10, 1000L, 2, 100));
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(new String[] { "default", "newClass" }, (short) 1, 10, 10, 1000L, 2, 100));
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedAccountStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedAccountStorageStatsByClusterName(anyString());
    doAnswer(invocation -> {
        String clusterName = invocation.getArgument(0);
        if (clusterName.equals(CLUSTER_NAME)) {
            return aggregatedPartitionClassStorageStats;
        } else {
            return null;
        }
    }).when(accountStatsStore).queryAggregatedPartitionClassStorageStatsByClusterName(anyString());
    ObjectMapper mapper = new ObjectMapper();
    // construct a request to get account stats
    JSONObject headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, CLUSTER_NAME);
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.ACCOUNT_REPORT.name());
    RestRequest request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    MockRestResponseChannel restResponseChannel = new MockRestResponseChannel();
    doOperation(request, restResponseChannel);
    assertEquals("Storage stats mismatch", aggregatedAccountStorageStats.getStorageStats(), mapper.readValue(restResponseChannel.getResponseBody(), AggregatedAccountStorageStats.class).getStorageStats());
    // construct a request to get partition class stats
    headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, CLUSTER_NAME);
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.PARTITION_CLASS_REPORT.name());
    request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    restResponseChannel = new MockRestResponseChannel();
    doOperation(request, restResponseChannel);
    assertEquals("Storage stats mismatch", aggregatedPartitionClassStorageStats.getStorageStats(), mapper.readValue(restResponseChannel.getResponseBody(), AggregatedPartitionClassStorageStats.class).getStorageStats());
    // test clustername not found case to ensure that it goes through the exception path
    headers = new JSONObject();
    headers.put(RestUtils.Headers.CLUSTER_NAME, "WRONG_CLUSTER");
    headers.put(RestUtils.Headers.GET_STATS_REPORT_TYPE, StatsReportType.ACCOUNT_REPORT.name());
    request = createRestRequest(RestMethod.GET, Operations.STATS_REPORT, headers, null);
    try {
        doOperation(request, new MockRestResponseChannel());
        fail("Operation should have failed");
    } catch (RestServiceException e) {
        assertEquals("ErrorCode not as expected", RestServiceErrorCode.NotFound, e.getErrorCode());
    }
}

Example 4 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method queryAggregatedPartitionClassStorageStatsByClusterName.

@Override
public AggregatedPartitionClassStorageStats queryAggregatedPartitionClassStorageStatsByClusterName(String clusterName) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(null);
    AtomicLong timestamp = new AtomicLong(0);
    partitionClassReportsDao.queryAggregatedPartitionClassReport(clusterName, (partitionClassName, accountId, containerStats, updatedAt) -> {
        aggregatedPartitionClassStorageStats.addContainerStorageStats(partitionClassName, accountId, containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAt));
    });
    storeMetrics.queryAggregatedPartitionClassStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return aggregatedPartitionClassStorageStats;
}

Example 5 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostPartitionClassStorageStatsWrappers.

/**
 * Aggregate all {@link HostPartitionClassStorageStatsWrapper} to generate two {@link AggregatedPartitionClassStorageStats}s. First
 * {@link AggregatedPartitionClassStorageStats} is the sum of all {@link HostPartitionClassStorageStatsWrapper}s. The second
 * {@link AggregatedPartitionClassStorageStats} is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostPartitionClassStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedPartitionClassStorageStats}.
 * @throws IOException
 */
Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregateHostPartitionClassStorageStatsWrappers(Map<String, HostPartitionClassStorageStatsWrapper> statsWrappers) throws IOException {
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> combinedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> selectedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostPartitionClassStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostPartitionClassStorageStatsWrapper hostPartitionClassStorageStatsWrapper = statsWrapperEntry.getValue();
        HostPartitionClassStorageStats hostPartitionClassStorageStats = hostPartitionClassStorageStatsWrapper.getStats();
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy1 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy2 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        combineRawHostPartitionClassStorageStatsMap(combinedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy1.getStorageStats());
        selectRawHostPartitionClassStorageStatsMap(selectedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostPartitionClassStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(combinedHostPartitionClassStorageStatsMap));
        logger.trace("Selected raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(selectedHostPartitionClassStorageStatsMap));
    }
    AggregatedPartitionClassStorageStats combinedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(combinedHostPartitionClassStorageStatsMap));
    AggregatedPartitionClassStorageStats selectedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(selectedHostPartitionClassStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}