Examples with StatsSnapshot com.github.ambry.server.StatsSnapshot used on opensource projects

Example 1 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregatorTest method generateNodeStats.

/**
 * Given a {@link List} of {@link StatsSnapshot}s and a timestamp generate a {@link StatsWrapper} that would have been
 * produced by a node.
 * @param storeSnapshots a {@link List} of store level {@link StatsSnapshot}s.
 * @param timestamp the timestamp to be attached to the generated {@link StatsWrapper}
 * @return the generated node level {@link StatsWrapper}
 */
private StatsWrapper generateNodeStats(List<StatsSnapshot> storeSnapshots, long timestamp) {
    Map<String, StatsSnapshot> partitionMap = new HashMap<>();
    long total = 0;
    int numbOfPartitions = storeSnapshots.size();
    for (int i = 0; i < numbOfPartitions; i++) {
        StatsSnapshot partitionSnapshot = storeSnapshots.get(i);
        partitionMap.put(String.format("partition_%d", i), partitionSnapshot);
        total += partitionSnapshot.getValue();
    }
    StatsSnapshot nodeSnapshot = new StatsSnapshot(total, partitionMap);
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.QUOTA, timestamp, numbOfPartitions, numbOfPartitions, Collections.EMPTY_LIST);
    return new StatsWrapper(header, nodeSnapshot);
}

Example 2 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregatorTest method generateStoreStats.

/**
 * Generate a quota {@link StatsSnapshot} based on the given parameters that would have been produced by a
 * {@link com.github.ambry.store.Store}.
 * @param accountCount number of account entry in the {@link StatsSnapshot}
 * @param containerCount number of container entry in the {@link StatsSnapshot}
 * @param random the random generator to be used
 * @return the generated store level {@link StatsSnapshot}
 */
private StatsSnapshot generateStoreStats(int accountCount, int containerCount, Random random) {
    Map<String, StatsSnapshot> accountMap = new HashMap<>();
    long totalSize = 0;
    for (int i = 0; i < accountCount; i++) {
        Map<String, StatsSnapshot> containerMap = new HashMap<>();
        long subTotalSize = 0;
        for (int j = 0; j < containerCount; j++) {
            long validSize = random.nextInt(2501) + 500;
            subTotalSize += validSize;
            containerMap.put(String.format("containerId_%d", j), new StatsSnapshot(validSize, null));
        }
        totalSize += subTotalSize;
        accountMap.put(String.format("accountId_%d", i), new StatsSnapshot(subTotalSize, containerMap));
    }
    return new StatsSnapshot(totalSize, accountMap);
}

Example 3 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregator method doWorkOnStatsWrapperMap.

Pair<StatsSnapshot, StatsSnapshot> doWorkOnStatsWrapperMap(Map<String, StatsWrapper> statsWrappers, StatsReportType type, boolean removeExceptionOnType) throws IOException {
    StatsSnapshot partitionSnapshot = new StatsSnapshot(0L, new HashMap<>());
    Map<String, Long> partitionTimestampMap = new HashMap<>();
    StatsSnapshot rawPartitionSnapshot = new StatsSnapshot(0L, new HashMap<>());
    if (removeExceptionOnType) {
        exceptionOccurredInstances.remove(type);
    }
    for (Map.Entry<String, StatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry != null && statsWrapperEntry.getValue() != null) {
            try {
                StatsWrapper snapshotWrapper = statsWrapperEntry.getValue();
                StatsWrapper snapshotWrapperCopy = new StatsWrapper(new StatsHeader(snapshotWrapper.getHeader()), new StatsSnapshot(snapshotWrapper.getSnapshot()));
                combineRawStats(rawPartitionSnapshot, snapshotWrapper);
                switch(type) {
                    case ACCOUNT_REPORT:
                        combineValidStatsByAccount(partitionSnapshot, snapshotWrapperCopy, statsWrapperEntry.getKey(), partitionTimestampMap);
                        break;
                    case PARTITION_CLASS_REPORT:
                        combineValidStatsByPartitionClass(partitionSnapshot, snapshotWrapperCopy, statsWrapperEntry.getKey(), partitionTimestampMap);
                        break;
                    default:
                        throw new IllegalArgumentException("Unrecognized stats report type: " + type);
                }
            } catch (Exception e) {
                logger.error("Exception occurred while processing stats from {}", statsWrapperEntry.getKey(), e);
                exceptionOccurredInstances.computeIfAbsent(type, key -> new ArrayList<>()).add(statsWrapperEntry.getKey());
            }
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw snapshot {}", mapper.writeValueAsString(rawPartitionSnapshot));
        logger.trace("Combined valid snapshot {}", mapper.writeValueAsString(partitionSnapshot));
    }
    StatsSnapshot reducedRawSnapshot;
    StatsSnapshot reducedSnapshot;
    switch(type) {
        case ACCOUNT_REPORT:
            reducedRawSnapshot = reduceByAccount(rawPartitionSnapshot);
            reducedSnapshot = reduceByAccount(partitionSnapshot);
            break;
        case PARTITION_CLASS_REPORT:
            reducedRawSnapshot = reduceByPartitionClass(rawPartitionSnapshot);
            reducedSnapshot = reduceByPartitionClass(partitionSnapshot);
            break;
        default:
            throw new IllegalArgumentException("Unrecognized stats report type: " + type);
    }
    reducedRawSnapshot.removeZeroValueSnapshots();
    reducedSnapshot.removeZeroValueSnapshots();
    if (logger.isTraceEnabled()) {
        logger.trace("Reduced raw snapshot {}", mapper.writeValueAsString(reducedRawSnapshot));
        logger.trace("Reduced valid snapshot {}", mapper.writeValueAsString(reducedSnapshot));
    }
    return new Pair<>(reducedRawSnapshot, reducedSnapshot);
}

Example 4 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregator method reduceByPartitionClass.

/**
 * Reduce the given {@link StatsSnapshot} whose first level mapped by PartitionClass to a shallower {@link StatsSnapshot}.
 * by adding entries belonging to each partition together and aggregating based on partition class. The partition level
 * would be removed after reduce work completes.
 * <pre>
 *             Before Reduce                   |             After Reduce
 * --------------------------------------------------------------------------------------------
 * {                                           |        {
 *   value: 1000,                              |          value:1000
 *   subMap:{                                  |          subMap:{
 *     PartitionClass_1: {                     |            PartitionClass_1: {
 *       value: 1000,                          |              value: 1000,
 *       subMap: {                             |              subMap: {
 *         Partition[1]:{                      |                Account[1]_Container[1]:{
 *           value: 400,                       |                  value: 1000,
 *           subMap: {                         |                  subMap: null
 *             Account[1]_Container[1]:{       |                }
 *               value: 400,                   |              }
 *               subMap: null                  |            }
 *             }                               |          }
 *           }                                 |        }
 *         },                                  |
 *         Partition[2]:{                      |
 *           value: 600,                       |
 *           subMap: {                         |
 *             Account[1]_Container[1]:{       |
 *               value: 600,                   |
 *               subMap: null                  |
 *             }                               |
 *           }                                 |
 *         }                                   |
 *       }
 *     }
 *   }
 * }
 * </pre>
 * @param statsSnapshot the {@link StatsSnapshot} to be reduced
 * @return the reduced {@link StatsSnapshot}
 */
static StatsSnapshot reduceByPartitionClass(StatsSnapshot statsSnapshot) {
    StatsSnapshot returnSnapshot = new StatsSnapshot(statsSnapshot.getValue(), new HashMap<>());
    Map<String, StatsSnapshot> partitionClassSnapshots = statsSnapshot.getSubMap();
    if (!partitionClassSnapshots.isEmpty()) {
        for (Map.Entry<String, StatsSnapshot> partitionClassToSnapshot : partitionClassSnapshots.entrySet()) {
            StatsSnapshot partitionClassSnapshot = partitionClassToSnapshot.getValue();
            StatsSnapshot reducedPartitionClassSnapshot = new StatsSnapshot(0L, null);
            Map<String, StatsSnapshot> partitionToSnapshot = partitionClassSnapshot.getSubMap();
            for (StatsSnapshot snapshot : partitionToSnapshot.values()) {
                StatsSnapshot.aggregate(reducedPartitionClassSnapshot, snapshot);
            }
            returnSnapshot.getSubMap().put(partitionClassToSnapshot.getKey(), reducedPartitionClassSnapshot);
        }
    }
    return returnSnapshot;
}

Example 5 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testFetchMonthlyStorageUsage.

/**
 * Test to update container storage usage monthly base.
 */
@Test
public void testFetchMonthlyStorageUsage() throws Exception {
    MockTime mockTime = new MockTime(SystemTime.getInstance().milliseconds());
    MySqlStorageUsageRefresher.time = mockTime;
    try {
        String currentMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        properties.remove(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS);
        StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
        AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
        MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
        // Fetch monthly storage usage
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month
        String notCurrentMonth = "1970-01";
        Map<String, Map<String, Long>> newContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(newContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(notCurrentMonth);
        refresher.fetchStorageUsageMonthlyBase();
        // Monthly storage usage still the old one
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month back to the current month
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        // Wait for schedule to retry
        Thread.sleep(MYSQL_RETRY_BACKOFF_MS * 2);
        assertEquals(newContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Forward the time to next month
        mockTime.sleep((MySqlStorageUsageRefresher.secondsToNextMonthTick(currentMonth, storageQuotaConfig.mysqlMonthlyBaseFetchOffsetSec) + 10) * 1000);
        String nextMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Function<String, Integer> stringMonthToInteger = (monthInStr) -> {
            String[] parts = monthInStr.split("-");
            int year = Integer.parseInt(parts[0]);
            int month = Integer.parseInt(parts[1]);
            return year * 12 + month;
        };
        assertEquals(stringMonthToInteger.apply(currentMonth) + 1, (int) stringMonthToInteger.apply(nextMonth));
        // Update the month to next month
        Map<String, Map<String, Long>> nextContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(nextContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(nextMonth);
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(nextContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // A backup file should be create as well
        assertEquals(nextContainerStorageUsages, refresher.getBackupFileManager().getBackupFileContent(nextMonth));
    } finally {
        MySqlStorageUsageRefresher.time = SystemTime.getInstance();
    }
}