Examples with AggregatedPartitionClassStorageStats com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats used on opensource projects

Example 6 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class MySqlReportAggregatorTask method removeInvalidAggregatedPartitionClassStats.

private void removeInvalidAggregatedPartitionClassStats(AggregatedPartitionClassStorageStats currentStats) throws Exception {
    List<Pair<String, Pair<Short, Short>>> toBeDeletedPartitionClassNameAndAccountContainer = new ArrayList<>();
    AggregatedPartitionClassStorageStats existingStats = accountStatsStore.queryAggregatedPartitionClassStorageStats();
    for (String partitionClassName : existingStats.getStorageStats().keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> currentAccountMap = currentStats.getStorageStats().get(partitionClassName);
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> accountEntry : existingStats.getStorageStats().get(partitionClassName).entrySet()) {
            short accountId = accountEntry.getKey();
            for (short containerId : accountEntry.getValue().keySet()) {
                if (currentAccountMap == null || !currentAccountMap.containsKey(accountId) || !currentAccountMap.get(accountId).containsKey(containerId)) {
                    toBeDeletedPartitionClassNameAndAccountContainer.add(new Pair<>(partitionClassName, new Pair<>(accountId, containerId)));
                }
            }
        }
    }
    for (Pair<String, Pair<Short, Short>> pair : toBeDeletedPartitionClassNameAndAccountContainer) {
        Pair<Short, Short> accountContainerId = pair.getSecond();
        accountStatsStore.deleteAggregatedPartitionClassStatsForAccountContainer(pair.getFirst(), accountContainerId.getFirst(), accountContainerId.getSecond());
    }
}

Example 7 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class StorageStatsUtilTest method testAggregatedPartitionClassStorageStatsConverter.

/**
 * Test case for {@link StorageStatsUtil#convertAggregatedPartitionClassStorageStatsToStatsSnapshot}.
 */
@Test
public void testAggregatedPartitionClassStorageStatsConverter() {
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(generateRandomAggregatedPartitionClassStorageStats(new String[] { "default", "newClass" }, (short) 100, 10, 10, 10000L, 2, 10));
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = aggregatedPartitionClassStorageStats.getStorageStats();
    StatsSnapshot expected = TestUtils.makePartitionClassSnasphotFromContainerStorageMap(convertAggregatedPartitionClassStorageStatsMapToContainerStorageMap(storageStats, false));
    StatsSnapshot snapshot = StorageStatsUtil.convertAggregatedPartitionClassStorageStatsToStatsSnapshot(aggregatedPartitionClassStorageStats, false);
    Assert.assertEquals(expected, snapshot);
    expected = TestUtils.makePartitionClassSnasphotFromContainerStorageMap(convertAggregatedPartitionClassStorageStatsMapToContainerStorageMap(storageStats, true));
    snapshot = StorageStatsUtil.convertAggregatedPartitionClassStorageStatsToStatsSnapshot(aggregatedPartitionClassStorageStats, true);
    Assert.assertEquals(expected, snapshot);
}

Example 8 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class StorageStatsTest method testAggregatedPartitionClassStorageStats.

@Test
public void testAggregatedPartitionClassStorageStats() throws Exception {
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> storageStatsMap = StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(new String[] { "default", "newClass" }, (short) 10, 10, 5, 10000L, 2, 100);
    String serialized = objectMapper.writeValueAsString(storageStatsMap);
    AggregatedPartitionClassStorageStats deserialized = objectMapper.readValue(serialized, AggregatedPartitionClassStorageStats.class);
    Assert.assertEquals(storageStatsMap, deserialized.getStorageStats());
    serialized = objectMapper.writeValueAsString(deserialized);
    deserialized = objectMapper.readValue(serialized, AggregatedPartitionClassStorageStats.class);
    Assert.assertEquals(storageStatsMap, deserialized.getStorageStats());
}

Example 9 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostPartitionClassStorageStatsWithOutdatedNode.

/**
 * Test {@link MySqlClusterAggregator#aggregateHostPartitionClassStorageStatsWrappers} but with one node having outdated
 * storage stats data. Outdated data shouldn't be used when aggregating valid storage stats.
 * @throws Exception
 */
@Test
public void testAggregateHostPartitionClassStorageStatsWithOutdatedNode() throws Exception {
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> upToDateStorageStatsMap = new HashMap<>();
    upToDateStorageStatsMap.computeIfAbsent("default", k -> new HashMap<>()).put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> outdatedStorageStatsMap = new HashMap<>();
    outdatedStorageStatsMap.computeIfAbsent("default", k -> new HashMap<>()).put((long) 0, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, 5, 3, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 1, 1, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper upToDateNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(upToDateStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, 0, 1, 1, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper outdatedNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(outdatedStorageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), 0, 0, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper emptyNodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats());
    Map<String, HostPartitionClassStorageStatsWrapper> instanceToStatsMap = new LinkedHashMap<>();
    instanceToStatsMap.put("Instance_0", upToDateNodeStats);
    instanceToStatsMap.put("Instance_1", outdatedNodeStats);
    instanceToStatsMap.put("Instance_2", emptyNodeStats);
    Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(instanceToStatsMap);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> expectedValid = clusterAggregator.aggregateHostPartitionClassStorageStats(upToDateStorageStatsMap);
    Assert.assertEquals(expectedValid, aggregatedRawAndValidStats.getSecond().getStorageStats());
}

Example 10 with AggregatedPartitionClassStorageStats

use of com.github.ambry.server.storagestats.AggregatedPartitionClassStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testAggregatedPartitionClassStorageStats.

@Test
public void testAggregatedPartitionClassStorageStats() throws Exception {
    testHostPartitionClassStorageStats();
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    // Now we should have partition class names and partition ids in database
    // Construct an aggregated partition class report
    AggregatedPartitionClassStorageStats aggregatedStats = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(partitionNameAndIds.keySet().toArray(new String[0]), (short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore.storeAggregatedPartitionClassStorageStats(aggregatedStats);
    partitionNameAndIds = mySqlStore3.queryPartitionNameAndIds();
    AggregatedPartitionClassStorageStats aggregatedStats3 = new AggregatedPartitionClassStorageStats(StorageStatsUtilTest.generateRandomAggregatedPartitionClassStorageStats(partitionNameAndIds.keySet().toArray(new String[0]), (short) 0, 10, 10, 10000L, 2, 10));
    mySqlStore3.storeAggregatedPartitionClassStorageStats(aggregatedStats3);
    AggregatedPartitionClassStorageStats obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(aggregatedStats.getStorageStats(), obtained.getStorageStats());
    assertEquals(mySqlStore.queryAggregatedPartitionClassStorageStatsByClusterName("random-cluster").getStorageStats().size(), 0);
    AggregatedPartitionClassStorageStats obtained3 = mySqlStore3.queryAggregatedPartitionClassStorageStats();
    assertEquals(aggregatedStats3.getStorageStats(), obtained3.getStorageStats());
    // Fetch StatsSnapshot
    StatsSnapshot obtainedSnapshot = mySqlStore.queryAggregatedPartitionClassStats();
    assertEquals(StorageStatsUtil.convertAggregatedPartitionClassStorageStatsToStatsSnapshot(obtained, false), obtainedSnapshot);
    // Change one value and store it to mysql database again
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(aggregatedStats.getStorageStats());
    ContainerStorageStats origin = newStorageStatsMap.get("default").get((short) 1).get((short) 1);
    newStorageStatsMap.get("default").get((short) 1).put((short) 1, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    mySqlStore.storeAggregatedPartitionClassStorageStats(new AggregatedPartitionClassStorageStats(newStorageStatsMap));
    obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(newStorageStatsMap, obtained.getStorageStats());
    // Delete some account and container
    short accountId = (short) 1;
    short containerId = (short) 1;
    for (String partitionClassName : partitionNameAndIds.keySet()) {
        mySqlStore.deleteAggregatedPartitionClassStatsForAccountContainer(partitionClassName, accountId, containerId);
        newStorageStatsMap.get(partitionClassName).get(accountId).remove(containerId);
    }
    obtained = mySqlStore.queryAggregatedPartitionClassStorageStats();
    assertEquals(newStorageStatsMap, obtained.getStorageStats());
    mySqlStore3.shutdown();
}