use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.
the class MySqlStorageUsageRefresherTest method testFetchMonthlyStorageUsage.
/**
* Test to update container storage usage monthly base.
*/
@Test
public void testFetchMonthlyStorageUsage() throws Exception {
MockTime mockTime = new MockTime(SystemTime.getInstance().milliseconds());
MySqlStorageUsageRefresher.time = mockTime;
try {
String currentMonth = MySqlStorageUsageRefresher.getCurrentMonth();
Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
properties.remove(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS);
StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
// Fetch monthly storage usage
refresher.fetchStorageUsageMonthlyBase();
assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
// Change the month
String notCurrentMonth = "1970-01";
Map<String, Map<String, Long>> newContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(newContainerStorageUsages);
accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(notCurrentMonth);
refresher.fetchStorageUsageMonthlyBase();
// Monthly storage usage still the old one
assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
// Change the month back to the current month
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
// Wait for schedule to retry
Thread.sleep(MYSQL_RETRY_BACKOFF_MS * 2);
assertEquals(newContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
// Forward the time to next month
mockTime.sleep((MySqlStorageUsageRefresher.secondsToNextMonthTick(currentMonth, storageQuotaConfig.mysqlMonthlyBaseFetchOffsetSec) + 10) * 1000);
String nextMonth = MySqlStorageUsageRefresher.getCurrentMonth();
Function<String, Integer> stringMonthToInteger = (monthInStr) -> {
String[] parts = monthInStr.split("-");
int year = Integer.parseInt(parts[0]);
int month = Integer.parseInt(parts[1]);
return year * 12 + month;
};
assertEquals(stringMonthToInteger.apply(currentMonth) + 1, (int) stringMonthToInteger.apply(nextMonth));
// Update the month to next month
Map<String, Map<String, Long>> nextContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(nextContainerStorageUsages);
accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(nextMonth);
refresher.fetchStorageUsageMonthlyBase();
assertEquals(nextContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
// A backup file should be create as well
assertEquals(nextContainerStorageUsages, refresher.getBackupFileManager().getBackupFileContent(nextMonth));
} finally {
MySqlStorageUsageRefresher.time = SystemTime.getInstance();
}
}
use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.
the class MySqlStorageUsageRefresherTest method testStartRefresher.
/**
* Tests to start refresher with or without back up files.
* @throws Exception
*/
@Test
public void testStartRefresher() throws Exception {
// Store something to mysql database as container usage and monthly container usage
Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
// use new mysql store, so we don't have connection conflict with the accountStatsMySqlStore.
AccountStatsMySqlStore newAccountStatsMysqlStore = createAccountStatsMySqlStore();
MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
// we should get an container storage usage full of zero
Map<String, Map<String, Long>> usage = refresher.getContainerStorageUsage();
assertContainerUsageMapAllZero(usage);
// we should have backup files now
Path backupDirPath = Paths.get(properties.getProperty(StorageQuotaConfig.BACKUP_FILE_DIR));
Path backupFilePath = backupDirPath.resolve(MySqlStorageUsageRefresher.getCurrentMonth());
Map<String, Map<String, Long>> backupContainerStorageUsages = new ObjectMapper().readValue(backupFilePath.toFile(), new TypeReference<Map<String, Map<String, Long>>>() {
});
assertEquals(containerStorageUsages, backupContainerStorageUsages);
// recreate a refresher, but change the monthly container usages, new refresher should load it from backup
containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
Map<String, Map<String, Long>> currentMonthlyStorageUsages = refresher.getContainerStorageUsageMonthlyBase();
assertEquals(backupContainerStorageUsages, currentMonthlyStorageUsages);
}
use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.
the class MySqlStorageUsageRefresherTest method testRefresherUpdateAndListener.
/**
* Test when updating container total usage.
* @throws Exception
*/
@Test
public void testRefresherUpdateAndListener() throws Exception {
Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
// Set polling interval to 2 seconds
properties.setProperty(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS, "2000");
StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
AtomicReference<Map<String, Map<String, Long>>> containerUsageRef = new AtomicReference<>(null);
AtomicReference<CountDownLatch> latchRef = new AtomicReference<>(null);
refresher.registerListener(containerStorageUsage -> {
containerUsageRef.set(containerStorageUsage);
latchRef.get().countDown();
});
// Keep storage usage unchanged, listener should get an all-zero map
accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
CountDownLatch latch1 = new CountDownLatch(1);
latchRef.set(latch1);
latch1.await(10, TimeUnit.SECONDS);
assertContainerUsageMapAllZero(containerUsageRef.get());
// Change some usage, listener should get
containerStorageUsages.get("1").compute("1", (k, v) -> v + 1L);
accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
CountDownLatch latch2 = new CountDownLatch(1);
latchRef.set(latch2);
latch2.await(10, TimeUnit.SECONDS);
assertEquals((long) containerUsageRef.get().get("1").get("1"), 1L);
Map<String, Map<String, Long>> clone = cloneMap(containerUsageRef.get());
clone.get("1").put("1", 0L);
assertContainerUsageMapAllZero(clone);
}
use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.
the class JSONStringStorageQuotaSourceTest method testJSONStringStorageQuotaSource.
@Test
public void testJSONStringStorageQuotaSource() throws Exception {
// Trick to create a string literal without escape.
String json = "{`10`: {`1`: 1000, `2`: 3000}, `20`: {`4`: 2000, `5`: 1000}, `30`: 4000, `40`: 5000, `50`: {`6`: 6000}}".replace("`", "\"");
Properties properties = new Properties();
properties.setProperty(StorageQuotaConfig.STORAGE_QUOTA_IN_JSON, json);
StorageQuotaConfig config = new StorageQuotaConfig(new VerifiableProperties(properties));
// Setting up accounts and account service
InMemAccountService accountService = new InMemAccountService(false, false);
Account account = new AccountBuilder((short) 10, "10", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 1, "1", Container.ContainerStatus.ACTIVE, "", (short) 10).build()).addOrUpdateContainer(new ContainerBuilder((short) 2, "2", Container.ContainerStatus.ACTIVE, "", (short) 10).build()).build();
accountService.updateAccounts(Collections.singleton(account));
account = new AccountBuilder((short) 20, "20", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 20).build()).addOrUpdateContainer(new ContainerBuilder((short) 5, "5", Container.ContainerStatus.ACTIVE, "", (short) 20).build()).build();
accountService.updateAccounts(Collections.singleton(account));
account = new AccountBuilder((short) 30, "30", Account.AccountStatus.ACTIVE, QuotaResourceType.ACCOUNT).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 30).build()).build();
accountService.updateAccounts(Collections.singleton(account));
account = new AccountBuilder((short) 40, "40", Account.AccountStatus.ACTIVE, QuotaResourceType.ACCOUNT).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 40).build()).build();
accountService.updateAccounts(Collections.singleton(account));
account = new AccountBuilder((short) 50, "50", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 6, "6", Container.ContainerStatus.ACTIVE, "", (short) 50).build()).build();
accountService.updateAccounts(Collections.singleton(account));
JSONStringStorageQuotaSource source = new JSONStringStorageQuotaSource(config, accountService);
QuotaResourceType resourceType = QuotaResourceType.CONTAINER;
Quota quota = source.getQuota(new QuotaResource("1000_1", resourceType), QuotaName.STORAGE_IN_GB);
assertNull(quota);
quota = source.getQuota(new QuotaResource("10_1", resourceType), QuotaName.STORAGE_IN_GB);
assertEquals(1000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("10_2", resourceType), QuotaName.STORAGE_IN_GB);
assertEquals(3000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("20_4", resourceType), QuotaName.STORAGE_IN_GB);
assertEquals(2000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("20_5", resourceType), QuotaName.STORAGE_IN_GB);
assertEquals(1000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("30", QuotaResourceType.ACCOUNT), QuotaName.STORAGE_IN_GB);
assertEquals(4000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("40", QuotaResourceType.ACCOUNT), QuotaName.STORAGE_IN_GB);
assertEquals(5000L, (long) quota.getQuotaValue());
quota = source.getQuota(new QuotaResource("50_6", resourceType), QuotaName.STORAGE_IN_GB);
assertEquals(6000L, (long) quota.getQuotaValue());
}
Aggregations