Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 1 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StorageStatsTest method testHostAccountStorageStats.

/**
 * Test methods in {@link HostAccountStorageStats}.
 * @throws Exception
 */
@Test
public void testHostAccountStorageStats() throws Exception {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = new HashMap<>();
    long partitionId = 10000L;
    short accountId = 0;
    short containerId = 100;
    int numberOfPartitions = 2;
    int numberOfAccounts = 2;
    int numberOfContainers = 2;
    for (int i = 0; i < numberOfPartitions; i++) {
        partitionId++;
        if (!storageStats.containsKey(partitionId)) {
            storageStats.put(partitionId, new HashMap<>());
        }
        for (int j = 0; j < numberOfAccounts; j++) {
            accountId++;
            if (!storageStats.get(partitionId).containsKey(accountId)) {
                storageStats.get(partitionId).put(accountId, new HashMap<>());
            }
            for (int k = 0; k < numberOfContainers; k++) {
                containerId++;
                if (!storageStats.get(partitionId).get(accountId).containsKey(containerId)) {
                    storageStats.get(partitionId).get(accountId).put(containerId, generateRandomContainerStorageStats(containerId));
                }
            }
        }
    }
    HostAccountStorageStats host1 = new HostAccountStorageStats(storageStats);
    HostAccountStorageStats host2 = new HostAccountStorageStats();
    for (Map.Entry<Long, Map<Short, Map<Short, ContainerStorageStats>>> partitionEntry : storageStats.entrySet()) {
        long pid = partitionEntry.getKey();
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> accountEntry : partitionEntry.getValue().entrySet()) {
            short aid = accountEntry.getKey();
            for (Map.Entry<Short, ContainerStorageStats> containerEntry : accountEntry.getValue().entrySet()) {
                host2.addContainerStorageStats(pid, aid, containerEntry.getValue());
            }
        }
    }
    Assert.assertEquals(host1.getStorageStats(), host2.getStorageStats());
    // Serialize the host account storage stats
    String serialized = objectMapper.writeValueAsString(host1);
    HostAccountStorageStats deserialized = objectMapper.readValue(serialized, HostAccountStorageStats.class);
    Assert.assertEquals(host1.getStorageStats(), deserialized.getStorageStats());
}

Example 2 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StorageStatsTest method testContainerStorageStats.

/**
 * Test methods in {@link ContainerStorageStats}.
 * @throws Exception
 */
@Test
public void testContainerStorageStats() throws Exception {
    short containerId = 10;
    long logicalStorageUsage = 10000;
    long physicalStorageUsage = 20000;
    long numberOfBlobs = 100;
    ContainerStorageStats stats = new ContainerStorageStats.Builder(containerId).logicalStorageUsage(logicalStorageUsage).physicalStorageUsage(physicalStorageUsage).numberOfBlobs(numberOfBlobs).build();
    assertContainerStorageStats(stats, containerId, logicalStorageUsage, physicalStorageUsage, numberOfBlobs);
    String serialized = objectMapper.writeValueAsString(stats);
    Map<String, Object> tempMap = objectMapper.readValue(serialized, new TypeReference<Map<String, Object>>() {
    });
    // We are only expecting "containerId", "logicalStorageUsage", "physicalStorageUsage" and "numberOfBlobs" in the serialized string
    Assert.assertEquals(4, tempMap.size());
    for (String key : new String[] { "containerId", "logicalStorageUsage", "physicalStorageUsage", "numberOfBlobs" }) {
        Assert.assertTrue(tempMap.containsKey(key));
    }
    ContainerStorageStats deserialized = objectMapper.readValue(serialized, ContainerStorageStats.class);
    Assert.assertEquals(stats, deserialized);
    ContainerStorageStats newStats = stats.add(deserialized);
    assertContainerStorageStats(stats, containerId, logicalStorageUsage, physicalStorageUsage, numberOfBlobs);
    assertContainerStorageStats(newStats, containerId, 2 * logicalStorageUsage, 2 * physicalStorageUsage, 2 * numberOfBlobs);
    serialized = "{`logicalStorageUsage`:1234, `physicalStorageUsage`:2345, `numberOfBlobs`: 12}".replace("`", "\"");
    try {
        objectMapper.readValue(serialized, ContainerStorageStats.class);
        Assert.fail("Missing container is should fail deserialization");
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof IllegalStateException);
    }
}

Example 3 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToHostAccountStorageStats.

/**
 * Convert a {@link StatsSnapshot} to {@link HostAccountStorageStats}. We assume the given {@link StatsSnapshot} follows
 * the proper format so we can construct a {@link HostAccountStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link HostAccountStorageStats}.
 */
public static HostAccountStorageStats convertStatsSnapshotToHostAccountStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats();
    Map<String, StatsSnapshot> partitionMap = // The map whose key is the partition id
    Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> partitionEntry : partitionMap.entrySet()) {
        long partitionId = Utils.partitionIdFromStatsPartitionKey(partitionEntry.getKey());
        Map<String, StatsSnapshot> accountMap = Optional.ofNullable(partitionEntry.getValue().getSubMap()).orElseGet(// The map whose key is the account id
        HashMap::new);
        for (Map.Entry<String, StatsSnapshot> accountEntry : accountMap.entrySet()) {
            short accountId = Utils.accountIdFromStatsAccountKey(accountEntry.getKey());
            Map<String, StatsSnapshot> containerMap = Optional.ofNullable(accountEntry.getValue().getSubMap()).orElseGet(HashMap::new);
            for (Map.Entry<String, StatsSnapshot> containerEntry : containerMap.entrySet()) {
                short containerId = Utils.containerIdFromStatsContainerKey(containerEntry.getKey());
                long logicalStorageUsage = containerEntry.getValue().getValue();
                hostAccountStorageStats.addContainerStorageStats(partitionId, accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
            }
        }
    }
    return hostAccountStorageStats;
}

Example 4 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StorageStatsUtil method convertStatsSnapshotToAggregatedPartitionClassStorageStats.

/**
 * Convert a {@link StatsSnapshot} to an {@link AggregatedPartitionClassStorageStats}. We assume the given {@link StatsSnapshot}
 * follows the proper format so we can construct an {@link AggregatedPartitionClassStorageStats} from it.
 * @param snapshot The {@link StatsSnapshot}.
 * @return The {@link AggregatedPartitionClassStorageStats}.
 */
public static AggregatedPartitionClassStorageStats convertStatsSnapshotToAggregatedPartitionClassStorageStats(StatsSnapshot snapshot) {
    if (snapshot == null) {
        return null;
    }
    AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats = new AggregatedPartitionClassStorageStats(null);
    Map<String, StatsSnapshot> partitionClassNameSubMap = Optional.ofNullable(snapshot.getSubMap()).orElseGet(HashMap::new);
    for (Map.Entry<String, StatsSnapshot> partitionClasNameSubMapEntry : partitionClassNameSubMap.entrySet()) {
        String partitionClassName = partitionClasNameSubMapEntry.getKey();
        Map<String, StatsSnapshot> accountContainerSubMap = Optional.ofNullable(partitionClasNameSubMapEntry.getValue().getSubMap()).orElseGet(HashMap::new);
        for (Map.Entry<String, StatsSnapshot> accountContainerSubMapEntry : accountContainerSubMap.entrySet()) {
            short[] accountContainerIds = Utils.accountContainerIdFromPartitionClassStatsKey(accountContainerSubMapEntry.getKey());
            short accountId = accountContainerIds[0];
            short containerId = accountContainerIds[1];
            long logicalStorageUsage = accountContainerSubMapEntry.getValue().getValue();
            aggregatedPartitionClassStorageStats.addContainerStorageStats(partitionClassName, accountId, new ContainerStorageStats(containerId, logicalStorageUsage, logicalStorageUsage, 0));
        }
    }
    return aggregatedPartitionClassStorageStats;
}

Example 5 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class BlobStoreStatsTest method getContainerStorageStats.

/**
 * Go over the referenceIndex to collect valid data size information per container. The result is used for
 * verification purposes.
 * @param deleteReferenceTimeInMs the reference time in ms until which deletes are relevant
 * @param expiryReferenceTimeInMs the reference time in ms until which expirations are relevant
 * @param deleteTombstoneStats a hashmap that tracks stats related delete tombstones in log segments.
 * @return a nested {@link Map} of serviceId to containerId to valid data size
 */
private Map<Short, Map<Short, ContainerStorageStats>> getContainerStorageStats(long deleteReferenceTimeInMs, long expiryReferenceTimeInMs, Map<String, Pair<AtomicLong, AtomicLong>> deleteTombstoneStats) {
    Map<Short, Map<Short, ContainerStorageStats>> containerStorageStats = new HashMap<>();
    Map<Short, Map<Short, Long>> validSizeMap = new HashMap<>();
    Map<Short, Map<Short, Long>> physicalSizeMap = new HashMap<>();
    Map<Short, Map<Short, Set<StoreKey>>> storeKeyMap = new HashMap<>();
    Pair<Set<MockId>, Set<MockId>> expiredDeletes = new Pair<>(new HashSet<>(), new HashSet<>());
    for (Offset indSegStartOffset : state.referenceIndex.keySet()) {
        state.getValidIndexEntriesForIndexSegment(indSegStartOffset, deleteReferenceTimeInMs, expiryReferenceTimeInMs, null, deleteTombstoneStats, expiredDeletes, true, (entry, isValid) -> {
            IndexValue indexValue = entry.getValue();
            if (indexValue.isPut() && isValid) {
                StatsUtils.updateNestedMapHelper(validSizeMap, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            }
            StatsUtils.updateNestedMapHelper(physicalSizeMap, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            storeKeyMap.computeIfAbsent(indexValue.getAccountId(), k -> new HashMap<>()).computeIfAbsent(indexValue.getContainerId(), k -> new HashSet<>()).add(entry.getKey());
        });
    }
    for (short accountId : validSizeMap.keySet()) {
        for (short containerId : validSizeMap.get(accountId).keySet()) {
            containerStorageStats.computeIfAbsent(accountId, k -> new HashMap<>()).put(containerId, new ContainerStorageStats(containerId, validSizeMap.get(accountId).get(containerId), physicalSizeMap.get(accountId).get(containerId), storeKeyMap.get(accountId).get(containerId).size()));
        }
    }
    return containerStorageStats;
}