Examples with StatsSnapshot com.github.ambry.server.StatsSnapshot used on opensource projects

Example 11 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method convertAccountStatsToPartitionClassStats.

private StatsWrapper convertAccountStatsToPartitionClassStats(StatsWrapper accountStats, Map<String, String> partitionKeyToClassName) {
    Map<String, StatsSnapshot> partitionClassSubMap = new HashMap<>();
    StatsSnapshot originHostStats = accountStats.getSnapshot();
    for (String partitionKey : originHostStats.getSubMap().keySet()) {
        StatsSnapshot originPartitionStats = originHostStats.getSubMap().get(partitionKey);
        String currentClassName = partitionKeyToClassName.get(partitionKey);
        StatsSnapshot partitionClassStats = partitionClassSubMap.computeIfAbsent(currentClassName, k -> new StatsSnapshot(0L, new HashMap<>()));
        Map<String, StatsSnapshot> accountContainerSubMap = new HashMap<>();
        for (String accountKey : originPartitionStats.getSubMap().keySet()) {
            for (Map.Entry<String, StatsSnapshot> containerEntry : originPartitionStats.getSubMap().get(accountKey).getSubMap().entrySet()) {
                String containerKey = containerEntry.getKey();
                StatsSnapshot containerStats = new StatsSnapshot(containerEntry.getValue());
                String accountContainerKey = Utils.partitionClassStatsAccountContainerKey(Utils.accountIdFromStatsAccountKey(accountKey), Utils.containerIdFromStatsContainerKey(containerKey));
                accountContainerSubMap.put(accountContainerKey, containerStats);
            }
        }
        long accountContainerValue = accountContainerSubMap.values().stream().mapToLong(StatsSnapshot::getValue).sum();
        StatsSnapshot partitionStats = new StatsSnapshot(accountContainerValue, accountContainerSubMap);
        partitionClassStats.getSubMap().put(partitionKey, partitionStats);
        partitionClassStats.setValue(partitionClassStats.getValue() + accountContainerValue);
    }
    return new StatsWrapper(new StatsHeader(accountStats.getHeader()), new StatsSnapshot(originHostStats.getValue(), partitionClassSubMap));
}

Example 12 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedPartitionClassStats.

/**
 * Test methods to store, delete and fetch aggregated partition class stats.
 * @throws Exception
 */
@Test
public void testAggregatedPartitionClassStats() throws Exception {
    testHostPartitionClassStats();
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    // Now we should have partition class names and partition ids in database
    // Construct an aggregated partition class report
    StatsSnapshot aggregated = TestUtils.makeAggregatedPartitionClassStats(partitionNameAndIds.keySet().toArray(new String[0]), 10, 10);
    mySqlStore.storeAggregatedPartitionClassStats(aggregated);
    partitionNameAndIds = mySqlStore3.queryPartitionNameAndIds();
    StatsSnapshot aggregated3 = TestUtils.makeAggregatedPartitionClassStats(partitionNameAndIds.keySet().toArray(new String[0]), 10, 10);
    mySqlStore3.storeAggregatedPartitionClassStats(aggregated3);
    StatsSnapshot obtained = mySqlStore.queryAggregatedPartitionClassStats();
    assertEquals(aggregated, obtained);
    assertEquals(mySqlStore.queryAggregatedPartitionClassStatsByClusterName("random-cluster").getSubMap().size(), 0);
    StatsSnapshot obtained3 = mySqlStore3.queryAggregatedPartitionClassStats();
    assertEquals(aggregated3, obtained3);
    // Change one value and store it to mysql database again
    StatsSnapshot newSnapshot = new StatsSnapshot(aggregated);
    newSnapshot.getSubMap().get("default").getSubMap().get(Utils.partitionClassStatsAccountContainerKey((short) 1, (short) 1)).setValue(1);
    newSnapshot.updateValue();
    mySqlStore.storeAggregatedPartitionClassStats(aggregated);
    obtained = mySqlStore.queryAggregatedPartitionClassStats();
    assertEquals(aggregated, obtained);
    // Delete some account and container
    newSnapshot = new StatsSnapshot(newSnapshot);
    short accountId = (short) 1;
    short containerId = (short) 1;
    String accountContainerKey = Utils.partitionClassStatsAccountContainerKey(accountId, containerId);
    for (String partitionClassName : partitionNameAndIds.keySet()) {
        mySqlStore.deleteAggregatedPartitionClassStatsForAccountContainer(partitionClassName, accountId, containerId);
        newSnapshot.getSubMap().get(partitionClassName).getSubMap().remove(accountContainerKey);
    }
    newSnapshot.updateValue();
    obtained = mySqlStore.queryAggregatedPartitionClassStats();
    assertEquals(newSnapshot, obtained);
    mySqlStore3.shutdown();
}

Example 13 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testFetchMonthlyStorageUsage.

/**
 * Test to update container storage usage monthly base.
 */
@Test
public void testFetchMonthlyStorageUsage() throws Exception {
    MockTime mockTime = new MockTime(SystemTime.getInstance().milliseconds());
    MySqlStorageUsageRefresher.time = mockTime;
    try {
        String currentMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        properties.remove(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS);
        StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
        AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
        MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
        // Fetch monthly storage usage
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month
        String notCurrentMonth = "1970-01";
        Map<String, Map<String, Long>> newContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(newContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(notCurrentMonth);
        refresher.fetchStorageUsageMonthlyBase();
        // Monthly storage usage still the old one
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month back to the current month
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        // Wait for schedule to retry
        Thread.sleep(MYSQL_RETRY_BACKOFF_MS * 2);
        assertEquals(newContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Forward the time to next month
        mockTime.sleep((MySqlStorageUsageRefresher.secondsToNextMonthTick(currentMonth, storageQuotaConfig.mysqlMonthlyBaseFetchOffsetSec) + 10) * 1000);
        String nextMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Function<String, Integer> stringMonthToInteger = (monthInStr) -> {
            String[] parts = monthInStr.split("-");
            int year = Integer.parseInt(parts[0]);
            int month = Integer.parseInt(parts[1]);
            return year * 12 + month;
        };
        assertEquals(stringMonthToInteger.apply(currentMonth) + 1, (int) stringMonthToInteger.apply(nextMonth));
        // Update the month to next month
        Map<String, Map<String, Long>> nextContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(nextContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(nextMonth);
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(nextContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // A backup file should be create as well
        assertEquals(nextContainerStorageUsages, refresher.getBackupFileManager().getBackupFileContent(nextMonth));
    } finally {
        MySqlStorageUsageRefresher.time = SystemTime.getInstance();
    }
}

Example 14 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testStartRefresher.

/**
 * Tests to start refresher with or without back up files.
 * @throws Exception
 */
@Test
public void testStartRefresher() throws Exception {
    // Store something to mysql database as container usage and monthly container usage
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    // use new mysql store, so we don't have connection conflict with the accountStatsMySqlStore.
    AccountStatsMySqlStore newAccountStatsMysqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    // we should get an container storage usage full of zero
    Map<String, Map<String, Long>> usage = refresher.getContainerStorageUsage();
    assertContainerUsageMapAllZero(usage);
    // we should have backup files now
    Path backupDirPath = Paths.get(properties.getProperty(StorageQuotaConfig.BACKUP_FILE_DIR));
    Path backupFilePath = backupDirPath.resolve(MySqlStorageUsageRefresher.getCurrentMonth());
    Map<String, Map<String, Long>> backupContainerStorageUsages = new ObjectMapper().readValue(backupFilePath.toFile(), new TypeReference<Map<String, Map<String, Long>>>() {
    });
    assertEquals(containerStorageUsages, backupContainerStorageUsages);
    // recreate a refresher, but change the monthly container usages, new refresher should load it from backup
    containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    Map<String, Map<String, Long>> currentMonthlyStorageUsages = refresher.getContainerStorageUsageMonthlyBase();
    assertEquals(backupContainerStorageUsages, currentMonthlyStorageUsages);
}

Example 15 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixAccountServiceTest method testSelectInactiveContainer.

/**
 * Tests select INACTIVE {@link Container}s from DELETE_IN_PROGRESS {@link Container}s.
 */
@Test
public void testSelectInactiveContainer() throws Exception {
    // generates store stats
    int accountCount = 1;
    int containerCount = 3;
    StatsSnapshot statsSnapshot = generateStoreStats(accountCount, containerCount, random, StatsReportType.ACCOUNT_REPORT);
    // a set that records the account ids that have already been taken.
    Set<Short> accountIdSet = new HashSet<>();
    // generate a single reference account and container that can be referenced by refAccount and refContainer respectively.
    refAccountId = Utils.getRandomShort(random);
    accountIdSet.add(refAccountId);
    generateRefAccounts(idToRefAccountMap, idToRefContainerMap, accountIdSet, 2, 3);
    accountService = mockHelixAccountServiceFactory.getAccountService();
    accountService.updateAccounts(idToRefAccountMap.values());
    assertAccountsInAccountService(idToRefAccountMap.values(), 2, accountService);
    Set<Container> expectContainerSet = new HashSet<>();
    List<Account> accountsToUpdate = new ArrayList<>();
    int accountId = 0;
    for (Account account : accountService.getAllAccounts()) {
        AccountBuilder accountBuilder = new AccountBuilder((short) accountId, "A[" + accountId + "]", AccountStatus.ACTIVE);
        int containerId = 0;
        for (Container container : account.getAllContainers()) {
            ContainerBuilder containerBuilder = new ContainerBuilder((short) containerId, "C[" + containerId + "]", ContainerStatus.DELETE_IN_PROGRESS, container.getDescription() + "--extra", (short) accountId);
            accountBuilder.addOrUpdateContainer(containerBuilder.build());
            containerId++;
        }
        accountsToUpdate.add(accountBuilder.build());
        if (accountId == 1) {
            expectContainerSet.addAll(accountsToUpdate.get(accountId).getAllContainers());
        }
        accountId++;
    }
    updateAccountsAndAssertAccountExistence(accountsToUpdate, 4, true);
    Set<Container> inactiveContainerSet = AccountUtils.selectInactiveContainerCandidates(statsSnapshot, accountService.getContainersByStatus(ContainerStatus.DELETE_IN_PROGRESS));
    assertEquals("Mismatch in container Set after detect", expectContainerSet, inactiveContainerSet);
    ((HelixAccountService) accountService).markContainersInactive(inactiveContainerSet);
    Account testAccount0 = accountService.getAccountById((short) 0);
    for (Container container : testAccount0.getAllContainers()) {
        assertEquals("Based on the stats report, container has not been compacted yet", ContainerStatus.DELETE_IN_PROGRESS, container.getStatus());
    }
    Account testAccount1 = accountService.getAccountById((short) 1);
    for (Container container : testAccount1.getAllContainers()) {
        assertEquals("Based on the stats report, inactive container status needs to be set as INACTIVE", ContainerStatus.INACTIVE, container.getStatus());
    }
}