Examples with HostAccountStorageStats com.github.ambry.server.storagestats.HostAccountStorageStats used on opensource projects

Example 6 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testStatsDeletePartitionAccountContainer.

/**
 * Test to delete partition, account and container data from database
 * @throws Exception
 */
@Test
public void testStatsDeletePartitionAccountContainer() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    // Now remove one partition from stats
    HostAccountStorageStats storageStatsCopy = new HostAccountStorageStats(stats.getStats());
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.remove((long) 1);
    HostAccountStorageStatsWrapper stats2 = new HostAccountStorageStatsWrapper(new StatsHeader(stats.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats2);
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats2.getStats().getStorageStats(), stats2.getStats().getStorageStats());
    // Now remove one account from stats
    storageStatsCopy = new HostAccountStorageStats(stats2.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    newStorageStatsMap.get((long) 3).remove((short) 1);
    HostAccountStorageStatsWrapper stats3 = new HostAccountStorageStatsWrapper(new StatsHeader(stats2.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats3);
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats3.getStats().getStorageStats(), stats3.getStats().getStorageStats());
    // Now remove some containers
    storageStatsCopy = new HostAccountStorageStats(stats3.getStats());
    newStorageStatsMap = new HashMap<>(storageStatsCopy.getStorageStats());
    for (short containerId : new short[] { 0, 1, 2 }) {
        newStorageStatsMap.get((long) 3).get((short) 3).remove(containerId);
    }
    HostAccountStorageStatsWrapper stats4 = new HostAccountStorageStatsWrapper(new StatsHeader(stats3.getHeader()), new HostAccountStorageStats(newStorageStatsMap));
    mySqlStore.storeHostAccountStorageStats(stats4);
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats4.getStats().getStorageStats(), stats4.getStats().getStorageStats());
    // Now write the stats back
    stats = generateHostAccountStorageStatsWrapper(10, 10, 10, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(stats.getStats().getStorageStats(), obtainedStats.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 7 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 still empty
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats2.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 not empty
    HostAccountStorageStatsWrapper stats3 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, stats.getStats().getStorageStats().get((long) 1));
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats3.getStats().getStorageStats().containsKey((long) 10));
    // Write an empty HostAccountStorageStats
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    // Empty storage stats should remove all the data in the database
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats4.getStats().getStorageStats().isEmpty());
    // Write an empty HostAccountStorageStats again
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    HostAccountStorageStatsWrapper obtainedStats5 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats5.getStats().getStorageStats().isEmpty());
    HostAccountStorageStatsWrapper stats6 = generateHostAccountStorageStatsWrapper(20, 20, 20, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats6);
    HostAccountStorageStatsWrapper obtainedStats6 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats6.getStats().getStorageStats(), stats6.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 8 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method queryHostAccountStorageStatsBySimplifiedHostName.

/**
 * Query mysql database to get all the container storage usage for given {@code clusterName} and {@code queryHostname} and
 * construct a {@link HostAccountStorageStatsWrapper} from them
 * @param queryHostname The simplified hostname to query.
 * @return {@link HostAccountStorageStatsWrapper} published by the given simplified hostname.
 * @throws SQLException
 */
private HostAccountStorageStatsWrapper queryHostAccountStorageStatsBySimplifiedHostName(String queryHostname) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats();
    AtomicLong timestamp = new AtomicLong(0);
    accountReportsDao.queryStorageUsageForHost(clusterName, queryHostname, (partitionId, accountId, containerStats, updatedAtMs) -> {
        hostAccountStorageStats.addContainerStorageStats(partitionId, accountId, containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAtMs));
    });
    storeMetrics.queryStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return new HostAccountStorageStatsWrapper(new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp.get(), hostAccountStorageStats.getStorageStats().size(), hostAccountStorageStats.getStorageStats().size(), null), hostAccountStorageStats);
}

Example 9 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method storeHostAccountStorageStats.

/**
 * Store the {@link HostAccountStorageStatsWrapper} to mysql database. This method ignores the error information from {@link HostAccountStorageStatsWrapper}
 * and only publish the container storage usages that are different from the previous one.
 * @param statsWrapper The {@link HostAccountStorageStatsWrapper} to publish.
 */
@Override
public void storeHostAccountStorageStats(HostAccountStorageStatsWrapper statsWrapper) throws Exception {
    AccountReportsDao.StorageBatchUpdater batch = accountReportsDao.new StorageBatchUpdater(config.updateBatchSize);
    int batchSize = 0;
    long startTimeMs = System.currentTimeMillis();
    HostAccountStorageStats prevHostStats = Optional.ofNullable(previousHostAccountStorageStatsWrapper).map(HostAccountStorageStatsWrapper::getStats).orElseGet(HostAccountStorageStats::new);
    HostAccountStorageStats currHostStats = statsWrapper.getStats();
    // Find the differences between two {@link HostAccountStorageStats} and apply them to the given {@link ContainerStorageStatsFunction}.
    // The difference is defined as
    // 1. If a container storage usage exists in both HostAccountStorageStats, and the values are different.
    // 2. If a container storage usage only exists in current HostAccountStorageStats.
    // If a container storage usage only exists in the previous HostAccountStorageStats, then it will not be applied to the given function.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> currPartitionMap = currHostStats.getStorageStats();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> prevPartitionMap = prevHostStats.getStorageStats();
    for (Map.Entry<Long, Map<Short, Map<Short, ContainerStorageStats>>> currPartitionMapEntry : currPartitionMap.entrySet()) {
        long partitionId = currPartitionMapEntry.getKey();
        Map<Short, Map<Short, ContainerStorageStats>> currAccountMap = currPartitionMapEntry.getValue();
        Map<Short, Map<Short, ContainerStorageStats>> prevAccountMap = prevPartitionMap.getOrDefault(partitionId, Collections.emptyMap());
        // as its value.
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> currAccountMapEntry : currAccountMap.entrySet()) {
            short accountId = currAccountMapEntry.getKey();
            Map<Short, ContainerStorageStats> currContainerMap = currAccountMapEntry.getValue();
            Map<Short, ContainerStorageStats> prevContainerMap = prevAccountMap.getOrDefault(accountId, Collections.emptyMap());
            for (Map.Entry<Short, ContainerStorageStats> currContainerMapEntry : currContainerMap.entrySet()) {
                short containerId = currContainerMapEntry.getKey();
                ContainerStorageStats currContainerStorageStats = currContainerMapEntry.getValue();
                if (!prevContainerMap.containsKey(containerId) || !currContainerStorageStats.equals(prevContainerMap.get(containerId))) {
                    batch.addUpdateToBatch(clusterName, hostname, (int) partitionId, accountId, currContainerStorageStats);
                    batchSize++;
                }
            }
        }
    }
    batch.flush();
    storeMetrics.batchSize.update(batchSize);
    storeMetrics.insertAccountStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    deleteContainerAccountStats(prevPartitionMap, currPartitionMap);
    storeMetrics.publishTimeMs.update(System.currentTimeMillis() - startTimeMs);
    previousHostAccountStorageStatsWrapper = statsWrapper;
    writeStatsToLocalBackupFile();
}

Example 10 with HostAccountStorageStats

use of com.github.ambry.server.storagestats.HostAccountStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreTest method testLocalBackupFile.

@Test
public void testLocalBackupFile() throws IOException {
    // First, make sure there is no local backup file.
    Path tempDir = Files.createTempDirectory("AccountStatsMySqlStoreTest");
    Path localBackupFilePath = tempDir.resolve("localbackup");
    Properties prop = new Properties();
    prop.setProperty(AccountStatsMySqlConfig.LOCAL_BACKUP_FILE_PATH, localBackupFilePath.toString());
    AccountStatsMySqlConfig accountStatsMySqlConfig = new AccountStatsMySqlConfig(new VerifiableProperties(prop));
    AccountStatsMySqlStore store = new AccountStatsMySqlStore(accountStatsMySqlConfig, mockDataSource, clusterName, hostname, null, new MetricRegistry());
    assertNull(store.getPreviousHostAccountStorageStatsWrapper());
    // Second, save a backup file.
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats(StorageStatsUtilTest.generateRandomHostAccountStorageStats(10, 10, 10, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, System.currentTimeMillis(), 10, 10, null);
    HostAccountStorageStatsWrapper statsWrapper = new HostAccountStorageStatsWrapper(header, hostAccountStorageStats);
    ObjectMapper objectMapper = new ObjectMapper();
    objectMapper.writeValue(localBackupFilePath.toFile(), statsWrapper);
    store = new AccountStatsMySqlStore(accountStatsMySqlConfig, mockDataSource, clusterName, hostname, null, new MetricRegistry());
    HostAccountStorageStatsWrapper backupWrapper = store.getPreviousHostAccountStorageStatsWrapper();
    assertNotNull(backupWrapper);
    assertStatsHeader(backupWrapper.getHeader(), 10, 10);
    Assert.assertEquals(hostAccountStorageStats.getStorageStats(), backupWrapper.getStats().getStorageStats());
}