Search in sources :

Example 1 with StorageQuotaConfig

use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testFetchMonthlyStorageUsage.

/**
 * Test to update container storage usage monthly base.
 */
@Test
public void testFetchMonthlyStorageUsage() throws Exception {
    MockTime mockTime = new MockTime(SystemTime.getInstance().milliseconds());
    MySqlStorageUsageRefresher.time = mockTime;
    try {
        String currentMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        properties.remove(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS);
        StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
        AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
        MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
        // Fetch monthly storage usage
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month
        String notCurrentMonth = "1970-01";
        Map<String, Map<String, Long>> newContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(newContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(notCurrentMonth);
        refresher.fetchStorageUsageMonthlyBase();
        // Monthly storage usage still the old one
        assertEquals(containerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Change the month back to the current month
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonth);
        // Wait for schedule to retry
        Thread.sleep(MYSQL_RETRY_BACKOFF_MS * 2);
        assertEquals(newContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // Forward the time to next month
        mockTime.sleep((MySqlStorageUsageRefresher.secondsToNextMonthTick(currentMonth, storageQuotaConfig.mysqlMonthlyBaseFetchOffsetSec) + 10) * 1000);
        String nextMonth = MySqlStorageUsageRefresher.getCurrentMonth();
        Function<String, Integer> stringMonthToInteger = (monthInStr) -> {
            String[] parts = monthInStr.split("-");
            int year = Integer.parseInt(parts[0]);
            int month = Integer.parseInt(parts[1]);
            return year * 12 + month;
        };
        assertEquals(stringMonthToInteger.apply(currentMonth) + 1, (int) stringMonthToInteger.apply(nextMonth));
        // Update the month to next month
        Map<String, Map<String, Long>> nextContainerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
        snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(nextContainerStorageUsages);
        accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
        accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(nextMonth);
        refresher.fetchStorageUsageMonthlyBase();
        assertEquals(nextContainerStorageUsages, refresher.getContainerStorageUsageMonthlyBase());
        // A backup file should be create as well
        assertEquals(nextContainerStorageUsages, refresher.getBackupFileManager().getBackupFileContent(nextMonth));
    } finally {
        MySqlStorageUsageRefresher.time = SystemTime.getInstance();
    }
}
Also used : HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) AccountStatsMySqlStoreFactory(com.github.ambry.accountstats.AccountStatsMySqlStoreFactory) TestUtils(com.github.ambry.utils.TestUtils) Map(java.util.Map) After(org.junit.After) SystemTime(com.github.ambry.utils.SystemTime) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TypeReference(com.fasterxml.jackson.core.type.TypeReference) Path(java.nio.file.Path) Before(org.junit.Before) AfterClass(org.junit.AfterClass) MetricRegistry(com.codahale.metrics.MetricRegistry) Properties(java.util.Properties) Files(java.nio.file.Files) AccountStatsMySqlStore(com.github.ambry.accountstats.AccountStatsMySqlStore) VerifiableProperties(com.github.ambry.config.VerifiableProperties) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Utils(com.github.ambry.utils.Utils) Test(org.junit.Test) AccountStatsMySqlConfig(com.github.ambry.config.AccountStatsMySqlConfig) TimeUnit(java.util.concurrent.TimeUnit) StorageQuotaConfig(com.github.ambry.config.StorageQuotaConfig) CountDownLatch(java.util.concurrent.CountDownLatch) MockTime(com.github.ambry.utils.MockTime) Paths(java.nio.file.Paths) StatsSnapshot(com.github.ambry.server.StatsSnapshot) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) Assert(org.junit.Assert) VerifiableProperties(com.github.ambry.config.VerifiableProperties) StorageQuotaConfig(com.github.ambry.config.StorageQuotaConfig) HashMap(java.util.HashMap) Map(java.util.Map) AccountStatsMySqlStore(com.github.ambry.accountstats.AccountStatsMySqlStore) MockTime(com.github.ambry.utils.MockTime) StatsSnapshot(com.github.ambry.server.StatsSnapshot) Test(org.junit.Test)

Example 2 with StorageQuotaConfig

use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testStartRefresher.

/**
 * Tests to start refresher with or without back up files.
 * @throws Exception
 */
@Test
public void testStartRefresher() throws Exception {
    // Store something to mysql database as container usage and monthly container usage
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    // use new mysql store, so we don't have connection conflict with the accountStatsMySqlStore.
    AccountStatsMySqlStore newAccountStatsMysqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    // we should get an container storage usage full of zero
    Map<String, Map<String, Long>> usage = refresher.getContainerStorageUsage();
    assertContainerUsageMapAllZero(usage);
    // we should have backup files now
    Path backupDirPath = Paths.get(properties.getProperty(StorageQuotaConfig.BACKUP_FILE_DIR));
    Path backupFilePath = backupDirPath.resolve(MySqlStorageUsageRefresher.getCurrentMonth());
    Map<String, Map<String, Long>> backupContainerStorageUsages = new ObjectMapper().readValue(backupFilePath.toFile(), new TypeReference<Map<String, Map<String, Long>>>() {
    });
    assertEquals(containerStorageUsages, backupContainerStorageUsages);
    // recreate a refresher, but change the monthly container usages, new refresher should load it from backup
    containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    refresher = new MySqlStorageUsageRefresher(newAccountStatsMysqlStore, scheduler, storageQuotaConfig, metrics);
    Map<String, Map<String, Long>> currentMonthlyStorageUsages = refresher.getContainerStorageUsageMonthlyBase();
    assertEquals(backupContainerStorageUsages, currentMonthlyStorageUsages);
}
Also used : Path(java.nio.file.Path) VerifiableProperties(com.github.ambry.config.VerifiableProperties) StorageQuotaConfig(com.github.ambry.config.StorageQuotaConfig) HashMap(java.util.HashMap) Map(java.util.Map) AccountStatsMySqlStore(com.github.ambry.accountstats.AccountStatsMySqlStore) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) StatsSnapshot(com.github.ambry.server.StatsSnapshot) Test(org.junit.Test)

Example 3 with StorageQuotaConfig

use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.

the class MySqlStorageUsageRefresherTest method testRefresherUpdateAndListener.

/**
 * Test when updating container total usage.
 * @throws Exception
 */
@Test
public void testRefresherUpdateAndListener() throws Exception {
    Map<String, Map<String, Long>> containerStorageUsages = TestUtils.makeStorageMap(10, 10, 100000, 1000);
    StatsSnapshot snapshot = TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages);
    accountStatsMySqlStore.storeAggregatedAccountStats(snapshot);
    accountStatsMySqlStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(MySqlStorageUsageRefresher.getCurrentMonth());
    // Set polling interval to 2 seconds
    properties.setProperty(StorageQuotaConfig.REFRESHER_POLLING_INTERVAL_MS, "2000");
    StorageQuotaConfig storageQuotaConfig = new StorageQuotaConfig(new VerifiableProperties(properties));
    AccountStatsMySqlStore newAccountStatsMySqlStore = createAccountStatsMySqlStore();
    MySqlStorageUsageRefresher refresher = new MySqlStorageUsageRefresher(newAccountStatsMySqlStore, scheduler, storageQuotaConfig, metrics);
    AtomicReference<Map<String, Map<String, Long>>> containerUsageRef = new AtomicReference<>(null);
    AtomicReference<CountDownLatch> latchRef = new AtomicReference<>(null);
    refresher.registerListener(containerStorageUsage -> {
        containerUsageRef.set(containerStorageUsage);
        latchRef.get().countDown();
    });
    // Keep storage usage unchanged, listener should get an all-zero map
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch1 = new CountDownLatch(1);
    latchRef.set(latch1);
    latch1.await(10, TimeUnit.SECONDS);
    assertContainerUsageMapAllZero(containerUsageRef.get());
    // Change some usage, listener should get
    containerStorageUsages.get("1").compute("1", (k, v) -> v + 1L);
    accountStatsMySqlStore.storeAggregatedAccountStats(TestUtils.makeAccountStatsSnapshotFromContainerStorageMap(containerStorageUsages));
    CountDownLatch latch2 = new CountDownLatch(1);
    latchRef.set(latch2);
    latch2.await(10, TimeUnit.SECONDS);
    assertEquals((long) containerUsageRef.get().get("1").get("1"), 1L);
    Map<String, Map<String, Long>> clone = cloneMap(containerUsageRef.get());
    clone.get("1").put("1", 0L);
    assertContainerUsageMapAllZero(clone);
}
Also used : VerifiableProperties(com.github.ambry.config.VerifiableProperties) AtomicReference(java.util.concurrent.atomic.AtomicReference) StorageQuotaConfig(com.github.ambry.config.StorageQuotaConfig) CountDownLatch(java.util.concurrent.CountDownLatch) AccountStatsMySqlStore(com.github.ambry.accountstats.AccountStatsMySqlStore) HashMap(java.util.HashMap) Map(java.util.Map) StatsSnapshot(com.github.ambry.server.StatsSnapshot) Test(org.junit.Test)

Example 4 with StorageQuotaConfig

use of com.github.ambry.config.StorageQuotaConfig in project ambry by linkedin.

the class JSONStringStorageQuotaSourceTest method testJSONStringStorageQuotaSource.

@Test
public void testJSONStringStorageQuotaSource() throws Exception {
    // Trick to create a string literal without escape.
    String json = "{`10`: {`1`: 1000, `2`: 3000}, `20`: {`4`: 2000, `5`: 1000}, `30`: 4000, `40`: 5000, `50`: {`6`: 6000}}".replace("`", "\"");
    Properties properties = new Properties();
    properties.setProperty(StorageQuotaConfig.STORAGE_QUOTA_IN_JSON, json);
    StorageQuotaConfig config = new StorageQuotaConfig(new VerifiableProperties(properties));
    // Setting up accounts and account service
    InMemAccountService accountService = new InMemAccountService(false, false);
    Account account = new AccountBuilder((short) 10, "10", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 1, "1", Container.ContainerStatus.ACTIVE, "", (short) 10).build()).addOrUpdateContainer(new ContainerBuilder((short) 2, "2", Container.ContainerStatus.ACTIVE, "", (short) 10).build()).build();
    accountService.updateAccounts(Collections.singleton(account));
    account = new AccountBuilder((short) 20, "20", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 20).build()).addOrUpdateContainer(new ContainerBuilder((short) 5, "5", Container.ContainerStatus.ACTIVE, "", (short) 20).build()).build();
    accountService.updateAccounts(Collections.singleton(account));
    account = new AccountBuilder((short) 30, "30", Account.AccountStatus.ACTIVE, QuotaResourceType.ACCOUNT).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 30).build()).build();
    accountService.updateAccounts(Collections.singleton(account));
    account = new AccountBuilder((short) 40, "40", Account.AccountStatus.ACTIVE, QuotaResourceType.ACCOUNT).addOrUpdateContainer(new ContainerBuilder((short) 4, "4", Container.ContainerStatus.ACTIVE, "", (short) 40).build()).build();
    accountService.updateAccounts(Collections.singleton(account));
    account = new AccountBuilder((short) 50, "50", Account.AccountStatus.ACTIVE, QuotaResourceType.CONTAINER).addOrUpdateContainer(new ContainerBuilder((short) 6, "6", Container.ContainerStatus.ACTIVE, "", (short) 50).build()).build();
    accountService.updateAccounts(Collections.singleton(account));
    JSONStringStorageQuotaSource source = new JSONStringStorageQuotaSource(config, accountService);
    QuotaResourceType resourceType = QuotaResourceType.CONTAINER;
    Quota quota = source.getQuota(new QuotaResource("1000_1", resourceType), QuotaName.STORAGE_IN_GB);
    assertNull(quota);
    quota = source.getQuota(new QuotaResource("10_1", resourceType), QuotaName.STORAGE_IN_GB);
    assertEquals(1000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("10_2", resourceType), QuotaName.STORAGE_IN_GB);
    assertEquals(3000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("20_4", resourceType), QuotaName.STORAGE_IN_GB);
    assertEquals(2000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("20_5", resourceType), QuotaName.STORAGE_IN_GB);
    assertEquals(1000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("30", QuotaResourceType.ACCOUNT), QuotaName.STORAGE_IN_GB);
    assertEquals(4000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("40", QuotaResourceType.ACCOUNT), QuotaName.STORAGE_IN_GB);
    assertEquals(5000L, (long) quota.getQuotaValue());
    quota = source.getQuota(new QuotaResource("50_6", resourceType), QuotaName.STORAGE_IN_GB);
    assertEquals(6000L, (long) quota.getQuotaValue());
}
Also used : Account(com.github.ambry.account.Account) InMemAccountService(com.github.ambry.account.InMemAccountService) ContainerBuilder(com.github.ambry.account.ContainerBuilder) QuotaResourceType(com.github.ambry.quota.QuotaResourceType) Quota(com.github.ambry.quota.Quota) VerifiableProperties(com.github.ambry.config.VerifiableProperties) AccountBuilder(com.github.ambry.account.AccountBuilder) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) StorageQuotaConfig(com.github.ambry.config.StorageQuotaConfig) QuotaResource(com.github.ambry.quota.QuotaResource) Test(org.junit.Test)

Aggregations

StorageQuotaConfig (com.github.ambry.config.StorageQuotaConfig)4 VerifiableProperties (com.github.ambry.config.VerifiableProperties)4 Test (org.junit.Test)4 AccountStatsMySqlStore (com.github.ambry.accountstats.AccountStatsMySqlStore)3 StatsSnapshot (com.github.ambry.server.StatsSnapshot)3 HashMap (java.util.HashMap)3 Map (java.util.Map)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 Path (java.nio.file.Path)2 Properties (java.util.Properties)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 MetricRegistry (com.codahale.metrics.MetricRegistry)1 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 Account (com.github.ambry.account.Account)1 AccountBuilder (com.github.ambry.account.AccountBuilder)1 ContainerBuilder (com.github.ambry.account.ContainerBuilder)1 InMemAccountService (com.github.ambry.account.InMemAccountService)1 AccountStatsMySqlStoreFactory (com.github.ambry.accountstats.AccountStatsMySqlStoreFactory)1 AccountStatsMySqlConfig (com.github.ambry.config.AccountStatsMySqlConfig)1