Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 11 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStoreMultilpleWrites.

/**
 * Tests to store multiple stats for one hosts and recover stats from database.
 * @throws Exception
 */
@Test
public void testStoreMultilpleWrites() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats1 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats1);
    HostAccountStorageStats hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    ContainerStorageStats origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats2.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).physicalStorageUsage(origin.getPhysicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats3);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats3.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).numberOfBlobs(origin.getNumberOfBlobs() + 1).build());
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats4);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats4.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 12 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method convertHostAccountStorageStatsToHostPartitionClassStorageStats.

private HostPartitionClassStorageStatsWrapper convertHostAccountStorageStatsToHostPartitionClassStorageStats(HostAccountStorageStatsWrapper accountStatsWrapper, Map<Long, String> partitionIdToClassName) {
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = accountStatsWrapper.getStats().getStorageStats();
    for (long partitionId : storageStats.keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> accountStorageStatsMap = storageStats.get(partitionId);
        String partitionClassName = partitionIdToClassName.get(partitionId);
        for (short accountId : accountStorageStatsMap.keySet()) {
            accountStorageStatsMap.get(accountId).values().forEach(containerStats -> hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, containerStats));
        }
    }
    return new HostPartitionClassStorageStatsWrapper(new StatsHeader(accountStatsWrapper.getHeader()), hostPartitionClassStorageStats);
}

Example 13 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStatsDeletePartitionAccountContainer.

/**
 * Test to delete partition, account and container data from database
 * @throws Exception
 */
@Test
public void testStatsDeletePartitionAccountContainer() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    // Now remove one partition from stats
    HostAccountStorageStats storageStatsCopy = new HostAccountStorageStats(stats.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.remove((long) 1);
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats2.getStats().getStorageStats(), stats2.getStats().getStorageStats());
    // Now remove one account from stats
    storageStatsCopy = new HostAccountStorageStats(stats2.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.get((long) 3).remove((short) 1);
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats2.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats3);
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats3.getStats().getStorageStats(), stats3.getStats().getStorageStats());
    // Now remove some containers
    storageStatsCopy = new HostAccountStorageStats(stats3.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    for (short containerId : new short[] { 0, 1, 2 }) {
        newStorageStatsMap.get((long) 3).get((short) 3).remove(containerId);
    }
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats3.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats4);
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats4.getStats().getStorageStats(), stats4.getStats().getStorageStats());
    // Now write the stats back
    stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 14 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 still empty
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats2.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 not empty
    HostAccountStorageStatsWrapper stats3 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, stats.getStats().getStorageStats().get((long) 1));
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats3.getStats().getStorageStats().containsKey((long) 10));
    // Write an empty HostAccountStorageStats
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    // Empty storage stats should remove all the data in the database
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats4.getStats().getStorageStats().isEmpty());
    // Write an empty HostAccountStorageStats again
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    HostAccountStorageStatsWrapper obtainedStats5 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats5.getStats().getStorageStats().isEmpty());
    HostAccountStorageStatsWrapper stats6 = generateHostAccountStorageStatsWrapper(20, 20, 20, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats6);
    HostAccountStorageStatsWrapper obtainedStats6 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats6.getStats().getStorageStats(), stats6.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 15 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class PartitionClassReportsDaoTest method testAggregatedPartitionClassReports.

@Test
public void testAggregatedPartitionClassReports() throws Exception {
    // Prepare the partition class name and partition id
    String clusterName = "ambry-test";
    String clusterName1 = "ambry-another-cluster";
    String className1 = "default";
    String className2 = "new-class";
    dao.insertPartitionClassName(clusterName, className1);
    dao.insertPartitionClassName(clusterName, className2);
    dao.insertPartitionClassName(clusterName1, className1);
    final Map<String, Map<Short, Map<Short, ContainerStorageStats>>> usagesInDB = new HashMap<>();
    dao.queryAggregatedPartitionClassReport(clusterName, (partitionClassName, accountId, containerStats, updatedAt) -> {
        usagesInDB.computeIfAbsent(partitionClassName, k -> new HashMap<>()).computeIfAbsent(accountId, k -> new HashMap<>()).put(containerStats.getContainerId(), containerStats);
    });
    assertTrue(usagesInDB.isEmpty());
    final int numAccount = 100;
    final int numContainer = 10;
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> classNameAccountContainerUsages = new HashMap<>();
    Random random = new Random();
    final long maxUsage = 10000;
    for (int i = 0; i < 3; i++) {
        classNameAccountContainerUsages.clear();
        usagesInDB.clear();
        for (String className : new String[] { className1, className2 }) {
            classNameAccountContainerUsages.put(className, IntStream.range(0, numAccount).boxed().collect(Collectors.toMap(Integer::shortValue, a -> IntStream.range(0, numContainer).boxed().collect(Collectors.toMap(Integer::shortValue, c -> new ContainerStorageStats.Builder(c.shortValue()).logicalStorageUsage(random.nextLong() % maxUsage).physicalStorageUsage(random.nextLong() % maxUsage).numberOfBlobs(10).build())))));
        }
        PartitionClassReportsDao.StorageBatchUpdater batch = dao.new StorageBatchUpdater(17);
        for (String className : classNameAccountContainerUsages.keySet()) {
            for (short accountId : classNameAccountContainerUsages.get(className).keySet()) {
                for (short containerId : classNameAccountContainerUsages.get(className).get(accountId).keySet()) {
                    ContainerStorageStats containerStats = classNameAccountContainerUsages.get(className).get(accountId).get(containerId);
                    batch.addUpdateToBatch(clusterName, className, accountId, containerStats);
                }
            }
        }
        batch.flush();
        dao.queryAggregatedPartitionClassReport(clusterName, (partitionClassName, accountId, containerStats, updatedAt) -> {
            usagesInDB.computeIfAbsent(partitionClassName, k -> new HashMap<>()).computeIfAbsent(accountId, k -> new HashMap<>()).put(containerStats.getContainerId(), containerStats);
        });
        assertEquals(classNameAccountContainerUsages, usagesInDB);
    }
}