Examples with HostAccountStorageStatsWrapper com.github.ambry.server.HostAccountStorageStatsWrapper used on opensource projects

Example 1 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithDifferentNumberOfStores.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers}, but with different numbers of partitions
 * from different nodes.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithDifferentNumberOfStores() throws Exception {
    // This storage stats has only 2 partitions, partition 0 and partition 1. Each partition has only one account and one container.
    String statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':10, 'physicalStorageUsage':20, 'numberOfBlobs':10}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':20, 'physicalStorageUsage':40, 'numberOfBlobs':20}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap1 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // This storage stats has only 3 partitions, partition 0, partition 1 and partition 2. Each partition has only one account and one container.
    statsInJson = "{'0': {'0': {'0': {'containerId':0, 'logicalStorageUsage':30, 'physicalStorageUsage':60, 'numberOfBlobs':30}}}, '1': {'0': {'0': {'containerId':0, 'logicalStorageUsage':40, 'physicalStorageUsage':80, 'numberOfBlobs':40}}}, '2': {'0': {'0': {'containerId':0, 'logicalStorageUsage':50, 'physicalStorageUsage':100, 'numberOfBlobs':50}}}}";
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap2 = objectMapper.readValue(statsInJson.replace("'", "\""), HostAccountStorageStats.class).getStorageStats();
    // Raw combined storage stats should only have one account and one container, and its storage stats is the sum of storage stats from all nodes.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':150, 'physicalStorageUsage':300, 'numberOfBlobs':150}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedRaw = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // Aggregated storage stats should only have one account and one container, and its storage stats is the sum of storage stats from second node,
    // since second node's physical storage usage is larger than first node at every partition.
    statsInJson = "{'0': {'0': {'containerId':0, 'logicalStorageUsage':120, 'physicalStorageUsage':240, 'numberOfBlobs':120}}}";
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = objectMapper.readValue(statsInJson.replace("'", "\""), new TypeReference<Map<Short, Map<Short, ContainerStorageStats>>>() {
    });
    // StorageStatsMap1 only have 2 store stats,  StorageStatsMap2 has 3
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 2, 2, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats1 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap1));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 3, 3, Collections.emptyList());
    HostAccountStorageStatsWrapper nodeStats2 = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(storageStatsMap2));
    Map<String, HostAccountStorageStatsWrapper> instanceStatsMap = new HashMap<>();
    instanceStatsMap.put("Instance_1", nodeStats1);
    instanceStatsMap.put("Instance_2", nodeStats2);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceStatsMap);
    Assert.assertEquals(expectedRaw, aggregatedRawAndValidStats.getFirst().getStorageStats());
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 2 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostAccountStorageStatsWithOutdatedNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostAccountStorageStatsWrappers} but with one node having outdated
 * storage stats data. Outdated data shouldn't be used when aggregating valid storage usage.
 * @throws Exception
 */
@Test
public void testAggregateHostAccountStorageStatsWithOutdatedNode() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> outdatedStorageStatsMap = new HashMap<>();
    outdatedStorageStatsMap.put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 6, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper upToDateNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostAccountStorageStatsWrapper outdatedNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats(outdatedStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 0, 0, Collections.emptyList());
    HostAccountStorageStatsWrapper emptyNodeStats = new HostAccountStorageStatsWrapper(header, new HostAccountStorageStats());
    Map<String, HostAccountStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", outdatedNodeStats);
    instanceToStatsMap.put("Instance_2", emptyNodeStats);
    Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostAccountStorageStatsWrappers(instanceToStatsMap);
    Map<Short, Map<Short, ContainerStorageStats>> expectedValid = clusterAggregator.aggregateHostAccountStorageStats(upToDateStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 3 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStoreMultilpleWrites.

/**
 * Tests to store multiple stats for one hosts and recover stats from database.
 * @throws Exception
 */
@Test
public void testStoreMultilpleWrites() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats1 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats1);
    HostAccountStorageStats hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    ContainerStorageStats origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats2.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).physicalStorageUsage(origin.getPhysicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats3);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats3.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).numberOfBlobs(origin.getNumberOfBlobs() + 1).build());
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats4);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats4.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 4 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testHostPartitionClassStorageStats.

/**
 * Test methods to store and fetch partition class, partition name partition id and partition class storage stats.
 * @throws Exception
 */
@Test
public void testHostPartitionClassStorageStats() throws Exception {
    // First write some stats to account reports
    testMultiStoreStats();
    HostAccountStorageStatsWrapper accountStats1 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    HostAccountStorageStatsWrapper accountStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname2, port2);
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    HostAccountStorageStatsWrapper accountStats3 = mySqlStore3.queryHostAccountStorageStatsByHost(hostname3, port3);
    // From this account stats, create partition class stats;
    Set<Long> allPartitionKeys = new HashSet<Long>() {

        {
            addAll(accountStats1.getStats().getStorageStats().keySet());
            addAll(accountStats2.getStats().getStorageStats().keySet());
            addAll(accountStats3.getStats().getStorageStats().keySet());
        }
    };
    List<String> partitionClassNames = Arrays.asList("default", "new");
    Map<Long, String> partitionIdToClassName = new HashMap<>();
    int ind = 0;
    for (long partitionId : allPartitionKeys) {
        partitionIdToClassName.put(partitionId, partitionClassNames.get(ind % partitionClassNames.size()));
        ind++;
    }
    HostPartitionClassStorageStatsWrapper partitionClassStats1 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats1, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats2 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats2, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats3 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats3, partitionIdToClassName);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats1);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats2);
    mySqlStore3.storeHostPartitionClassStorageStats(partitionClassStats3);
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    assertEquals(new HashSet<>(partitionClassNames), partitionNameAndIds.keySet());
    Map<Long, String> dbPartitionKeyToClassName = partitionNameAndIds.entrySet().stream().flatMap(ent -> ent.getValue().stream().map(pid -> new Pair<>(ent.getKey(), (long) pid))).collect(Collectors.toMap(Pair::getSecond, Pair::getFirst));
    assertEquals(partitionIdToClassName, dbPartitionKeyToClassName);
    // Fetch HostPartitionClassStorageStats
    HostPartitionClassStorageStatsWrapper obtainedStats1 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(partitionClassStats1.getStats().getStorageStats(), obtainedStats1.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname2, port2, partitionNameAndIds);
    assertEquals(partitionClassStats2.getStats().getStorageStats(), obtainedStats2.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats3 = mySqlStore3.queryHostPartitionClassStorageStatsByHost(hostname3, port3, partitionNameAndIds);
    assertEquals(partitionClassStats3.getStats().getStorageStats(), obtainedStats3.getStats().getStorageStats());
    // Fetch StatsSnapshot
    StatsWrapper obtainedStats = mySqlStore.queryPartitionClassStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(StorageStatsUtil.convertHostPartitionClassStorageStatsToStatsSnapshot(obtainedStats1.getStats(), false), obtainedStats.getSnapshot());
    mySqlStore3.shutdown();
}

Example 5 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStatsDeletePartitionAccountContainer.

/**
 * Test to delete partition, account and container data from database
 * @throws Exception
 */
@Test
public void testStatsDeletePartitionAccountContainer() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    // Now remove one partition from stats
    HostAccountStorageStats storageStatsCopy = new HostAccountStorageStats(stats.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.remove((long) 1);
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats2.getStats().getStorageStats(), stats2.getStats().getStorageStats());
    // Now remove one account from stats
    storageStatsCopy = new HostAccountStorageStats(stats2.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.get((long) 3).remove((short) 1);
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats2.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats3);
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats3.getStats().getStorageStats(), stats3.getStats().getStorageStats());
    // Now remove some containers
    storageStatsCopy = new HostAccountStorageStats(stats3.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    for (short containerId : new short[] { 0, 1, 2 }) {
        newStorageStatsMap.get((long) 3).get((short) 3).remove(containerId);
    }
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats3.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats4);
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats4.getStats().getStorageStats(), stats4.getStats().getStorageStats());
    // Now write the stats back
    stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}