Examples with StatsHeader com.github.ambry.server.StatsHeader used on opensource projects

Example 6 with StatsHeader

use of com.github.ambry.server.StatsHeader in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStoreMultilpleWrites.

/**
 * Tests to store multiple stats for one hosts and recover stats from database.
 * @throws Exception
 */
@Test
public void testStoreMultilpleWrites() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats1 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats1);
    HostAccountStorageStats hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    ContainerStorageStats origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).logicalStorageUsage(origin.getLogicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats2.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).physicalStorageUsage(origin.getPhysicalStorageUsage() + 1).build());
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats3);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats3.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    hostAccountStorageStatsCopy = new HostAccountStorageStats(stats1.getStats());
    newStorageStats = new HashMap<>(hostAccountStorageStatsCopy.getStorageStats());
    origin = newStorageStats.get((long) 0).get((short) 0).get((short) 0);
    newStorageStats.get((long) 0).get((short) 0).put((short) 0, new ContainerStorageStats.Builder(origin).numberOfBlobs(origin.getNumberOfBlobs() + 1).build());
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats1.getHeader()), new HostAccountStorageStats(newStorageStats));
    mySqlStore.storeHostAccountStorageStats(stats4);
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats4.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 7 with StatsHeader

use of com.github.ambry.server.StatsHeader in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method convertAccountStatsToPartitionClassStats.

private StatsWrapper convertAccountStatsToPartitionClassStats(StatsWrapper accountStats, Map<String, String> partitionKeyToClassName) {
    Map<String, StatsSnapshot> partitionClassSubMap = new HashMap<>();
    StatsSnapshot originHostStats = accountStats.getSnapshot();
    for (String partitionKey : originHostStats.getSubMap().keySet()) {
        StatsSnapshot originPartitionStats = originHostStats.getSubMap().get(partitionKey);
        String currentClassName = partitionKeyToClassName.get(partitionKey);
        StatsSnapshot partitionClassStats = partitionClassSubMap.computeIfAbsent(currentClassName, k -> new StatsSnapshot(0L, new HashMap<>()));
        Map<String, StatsSnapshot> accountContainerSubMap = new HashMap<>();
        for (String accountKey : originPartitionStats.getSubMap().keySet()) {
            for (Map.Entry<String, StatsSnapshot> containerEntry : originPartitionStats.getSubMap().get(accountKey).getSubMap().entrySet()) {
                String containerKey = containerEntry.getKey();
                StatsSnapshot containerStats = new StatsSnapshot(containerEntry.getValue());
                String accountContainerKey = Utils.partitionClassStatsAccountContainerKey(Utils.accountIdFromStatsAccountKey(accountKey), Utils.containerIdFromStatsContainerKey(containerKey));
                accountContainerSubMap.put(accountContainerKey, containerStats);
            }
        }
        long accountContainerValue = accountContainerSubMap.values().stream().mapToLong(StatsSnapshot::getValue).sum();
        StatsSnapshot partitionStats = new StatsSnapshot(accountContainerValue, accountContainerSubMap);
        partitionClassStats.getSubMap().put(partitionKey, partitionStats);
        partitionClassStats.setValue(partitionClassStats.getValue() + accountContainerValue);
    }
    return new StatsWrapper(new StatsHeader(accountStats.getHeader()), new StatsSnapshot(originHostStats.getValue(), partitionClassSubMap));
}

Example 8 with StatsHeader

use of com.github.ambry.server.StatsHeader in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method convertHostAccountStorageStatsToHostPartitionClassStorageStats.

private HostPartitionClassStorageStatsWrapper convertHostAccountStorageStatsToHostPartitionClassStorageStats(HostAccountStorageStatsWrapper accountStatsWrapper, Map<Long, String> partitionIdToClassName) {
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = accountStatsWrapper.getStats().getStorageStats();
    for (long partitionId : storageStats.keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> accountStorageStatsMap = storageStats.get(partitionId);
        String partitionClassName = partitionIdToClassName.get(partitionId);
        for (short accountId : accountStorageStatsMap.keySet()) {
            accountStorageStatsMap.get(accountId).values().forEach(containerStats -> hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, containerStats));
        }
    }
    return new HostPartitionClassStorageStatsWrapper(new StatsHeader(accountStatsWrapper.getHeader()), hostPartitionClassStorageStats);
}

Example 9 with StatsHeader

use of com.github.ambry.server.StatsHeader in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStatsDeletePartitionAccountContainer.

/**
 * Test to delete partition, account and container data from database
 * @throws Exception
 */
@Test
public void testStatsDeletePartitionAccountContainer() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    // Now remove one partition from stats
    HostAccountStorageStats storageStatsCopy = new HostAccountStorageStats(stats.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.remove((long) 1);
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats2.getStats().getStorageStats(), stats2.getStats().getStorageStats());
    // Now remove one account from stats
    storageStatsCopy = new HostAccountStorageStats(stats2.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.get((long) 3).remove((short) 1);
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats2.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats3);
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats3.getStats().getStorageStats(), stats3.getStats().getStorageStats());
    // Now remove some containers
    storageStatsCopy = new HostAccountStorageStats(stats3.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    for (short containerId : new short[] { 0, 1, 2 }) {
        newStorageStatsMap.get((long) 3).get((short) 3).remove(containerId);
    }
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats3.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats4);
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats4.getStats().getStorageStats(), stats4.getStats().getStorageStats());
    // Now write the stats back
    stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 10 with StatsHeader

use of com.github.ambry.server.StatsHeader in project ambry by linkedin.

the class AccountStatsMySqlStore method queryHostAccountStorageStatsBySimplifiedHostName.

/**
 * Query mysql database to get all the container storage usage for given {@code clusterName} and {@code queryHostname} and
 * construct a {@link HostAccountStorageStatsWrapper} from them
 * @param queryHostname The simplified hostname to query.
 * @return {@link HostAccountStorageStatsWrapper} published by the given simplified hostname.
 * @throws SQLException
 */
private HostAccountStorageStatsWrapper queryHostAccountStorageStatsBySimplifiedHostName(String queryHostname) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats();
    AtomicLong timestamp = new AtomicLong(0);
    accountReportsDao.queryStorageUsageForHost(clusterName, queryHostname, (partitionId, accountId, containerStats, updatedAtMs) -> {
        hostAccountStorageStats.addContainerStorageStats(partitionId, accountId, containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAtMs));
    });
    storeMetrics.queryStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return new HostAccountStorageStatsWrapper(new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp.get(), hostAccountStorageStats.getStorageStats().size(), hostAccountStorageStats.getStorageStats().size(), null), hostAccountStorageStats);
}