Examples with AggregatedAccountStorageStats com.github.ambry.server.storagestats.AggregatedAccountStorageStats used on opensource projects

Example 6 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostAccountStorageStatsWrappers.

/**
 * Aggregate all {@link HostAccountStorageStatsWrapper} to generate two {@link AggregatedAccountStorageStats}s. First
 * {@link AggregatedAccountStorageStats} is the sum of all {@link HostAccountStorageStatsWrapper}s. The second {@link AggregatedAccountStorageStats}
 * is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostAccountStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedAccountStorageStats}.
 * @throws IOException
 */
Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregateHostAccountStorageStatsWrappers(Map<String, HostAccountStorageStatsWrapper> statsWrappers) throws IOException {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> combinedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> selectedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostAccountStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostAccountStorageStatsWrapper hostAccountStorageStatsWrapper = statsWrapperEntry.getValue();
        HostAccountStorageStats hostAccountStorageStats = hostAccountStorageStatsWrapper.getStats();
        HostAccountStorageStats hostAccountStorageStatsCopy1 = new HostAccountStorageStats(hostAccountStorageStats);
        HostAccountStorageStats hostAccountStorageStatsCopy2 = new HostAccountStorageStats(hostAccountStorageStats);
        combineRawHostAccountStorageStatsMap(combinedHostAccountStorageStatsMap, hostAccountStorageStatsCopy1.getStorageStats());
        selectRawHostAccountStorageStatsMap(selectedHostAccountStorageStatsMap, hostAccountStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostAccountStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostAccountStorageStats {}", mapper.writeValueAsString(combinedHostAccountStorageStatsMap));
        logger.trace("Selected raw HostAccountStorageStats {}", mapper.writeValueAsString(selectedHostAccountStorageStatsMap));
    }
    AggregatedAccountStorageStats combinedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(combinedHostAccountStorageStatsMap));
    AggregatedAccountStorageStats selectedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(selectedHostAccountStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}

Example 7 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlReportAggregatorTask method removeInvalidAggregatedAccountAndContainerStats.

private void removeInvalidAggregatedAccountAndContainerStats(AggregatedAccountStorageStats currentStats) throws Exception {
    AggregatedAccountStorageStats existingStats = accountStatsStore.queryAggregatedAccountStorageStats();
    List<Pair<Short, Short>> toBeDeletedAccountAndContainers = new ArrayList<>();
    for (Map.Entry<Short, Map<Short, ContainerStorageStats>> accountEntry : existingStats.getStorageStats().entrySet()) {
        short accountId = accountEntry.getKey();
        for (short containerId : accountEntry.getValue().keySet()) {
            if (!currentStats.getStorageStats().containsKey(accountId) || !currentStats.getStorageStats().get(accountId).containsKey(containerId)) {
                toBeDeletedAccountAndContainers.add(new Pair<>(accountId, containerId));
            }
        }
    }
    // delete the account/container stats that are no longer valid
    for (Pair<Short, Short> accountContainer : toBeDeletedAccountAndContainers) {
        accountStatsStore.deleteAggregatedAccountStatsForContainer(accountContainer.getFirst(), accountContainer.getSecond());
    }
}

Example 8 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class MySqlReportAggregatorTask method run.

@Override
public TaskResult run() {
    Exception exception = null;
    Histogram fetchTimeMs = statsReportType == StatsReportType.ACCOUNT_REPORT ? metrics.accountStatsFetchTimeMs : metrics.partitionClassStatsFetchTimeMs;
    Histogram aggregationTimeMs = statsReportType == StatsReportType.ACCOUNT_REPORT ? metrics.accountStatsAggregationTimeMs : metrics.partitionClassStatsAggregationTimeMs;
    long startTimeMs = System.currentTimeMillis();
    StatsSnapshot accountPhysicalStorageSnapshot = null;
    try {
        List<String> instanceNames = manager.getClusterManagmentTool().getInstancesInCluster(manager.getClusterName());
        if (statsReportType == StatsReportType.ACCOUNT_REPORT) {
            Map<String, HostAccountStorageStatsWrapper> accountStatsWrappers = fetchAccountStorageStatsWrapperForInstances(instanceNames);
            fetchTimeMs.update(System.currentTimeMillis() - startTimeMs);
            logger.info("Aggregating stats from " + accountStatsWrappers.size() + " hosts");
            Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> results = clusterAggregator.aggregateHostAccountStorageStatsWrappers(accountStatsWrappers);
            if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                removeInvalidAggregatedAccountAndContainerStats(results.getSecond());
            }
            accountStatsStore.storeAggregatedAccountStorageStats(results.getSecond());
            accountPhysicalStorageSnapshot = StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(results.getFirst(), true);
        } else if (statsReportType == StatsReportType.PARTITION_CLASS_REPORT) {
            Map<String, HostPartitionClassStorageStatsWrapper> statsWrappers = fetchPartitionClassStorageStatsWrapperForInstances(instanceNames);
            fetchTimeMs.update(System.currentTimeMillis() - startTimeMs);
            logger.info("Aggregating stats from " + statsWrappers.size() + " hosts");
            Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> results = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(statsWrappers);
            if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                removeInvalidAggregatedPartitionClassStats(results.getSecond());
            }
            accountStatsStore.storeAggregatedPartitionClassStorageStats(results.getSecond());
        }
        // Check if there is a base report for this month or not.
        if (clusterMapConfig.clustermapEnableAggregatedMonthlyAccountReport && statsReportType == StatsReportType.ACCOUNT_REPORT) {
            // Get the month, if not the same month, then copy the aggregated stats and update the month
            String currentMonthValue = LocalDateTime.ofEpochSecond(time.seconds(), 0, ZONE_OFFSET).format(TIMESTAMP_FORMATTER);
            String recordedMonthValue = accountStatsStore.queryRecordedMonth();
            if (recordedMonthValue == null || recordedMonthValue.isEmpty() || !currentMonthValue.equals(recordedMonthValue)) {
                if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                    accountStatsStore.deleteSnapshotOfAggregatedAccountStats();
                }
                logger.info("Taking snapshot of aggregated stats for month " + currentMonthValue);
                accountStatsStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonthValue);
            }
        }
        aggregationTimeMs.update(System.currentTimeMillis() - startTimeMs);
        return new TaskResult(TaskResult.Status.COMPLETED, "Aggregation success");
    } catch (Exception e) {
        logger.error("Exception thrown while aggregating stats from container stats reports across all nodes ", e);
        exception = e;
        return new TaskResult(TaskResult.Status.FAILED, "Exception thrown");
    } finally {
        if (clusterMapConfig.clustermapEnableContainerDeletionAggregation && callback != null && accountPhysicalStorageSnapshot != null && statsReportType.equals(StatsReportType.ACCOUNT_REPORT)) {
            callback.onCompletion(accountPhysicalStorageSnapshot, exception);
        }
    }
}

Example 9 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToAggregatedAccountStorageStats.

/**
 * Convert a {@link StatsSnapshot} to {@link AggregatedAccountStorageStats}. We assume the given {@link StatsSnapshot}
 * follows the proper format so we can construct an {@link AggregatedAccountStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link AggregatedAccountStorageStats}.
 */
public static AggregatedAccountStorageStats convertStatsSnapshotToAggregatedAccountStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    AggregatedAccountStorageStats aggregatedAccountStorageStats = new AggregatedAccountStorageStats(null);
    Map<String, StatsSnapshot> accountMap = Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> accountMapEntry : accountMap.entrySet()) {
        short accountId = Utils.accountIdFromStatsAccountKey(accountMapEntry.getKey());
        Map<String, StatsSnapshot> containerMap = Optional.ofNullable(accountMapEntry.getValue().getSubMap()).orElseGet(HashMap::new);
        for (Map.Entry<String, StatsSnapshot> containerMapEntry : containerMap.entrySet()) {
            short containerId = Utils.containerIdFromStatsContainerKey(containerMapEntry.getKey());
            long logicalStorageUsage = containerMapEntry.getValue().getValue();
            aggregatedAccountStorageStats.addContainerStorageStats(accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
        }
    }
    return aggregatedAccountStorageStats;
}

Example 10 with AggregatedAccountStorageStats

use of com.github.ambry.server.storagestats.AggregatedAccountStorageStats in project ambry by linkedin.

the class StorageStatsTest method testAggregatedAccountStorageStats.

@Test
public void testAggregatedAccountStorageStats() throws Exception {
    Map<Short, Map<Short, ContainerStorageStats>> storageStatsMap = StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 10, 10, 5, 10000L, 2, 10);
    String serialized = objectMapper.writeValueAsString(storageStatsMap);
    AggregatedAccountStorageStats deserialized = objectMapper.readValue(serialized, AggregatedAccountStorageStats.class);
    Assert.assertEquals(storageStatsMap, deserialized.getStorageStats());
    serialized = objectMapper.writeValueAsString(deserialized);
    deserialized = objectMapper.readValue(serialized, AggregatedAccountStorageStats.class);
    Assert.assertEquals(storageStatsMap, deserialized.getStorageStats());
}