Examples with HostAccountStorageStatsWrapper com.github.ambry.server.HostAccountStorageStatsWrapper used on opensource projects

Example 6 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromLocalBackUpFile() throws Exception {
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 still empty
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats2.getStats().getStorageStats().containsKey((long) 10));
    // Write a new stats with partition 10 not empty
    HostAccountStorageStatsWrapper stats3 = generateHostAccountStorageStatsWrapper(10, 10, 1, StatsReportType.ACCOUNT_REPORT);
    newStorageStats = new HashMap<>(stats.getStats().getStorageStats());
    newStorageStats.put((long) 10, stats.getStats().getStorageStats().get((long) 1));
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats(newStorageStats)));
    HostAccountStorageStatsWrapper obtainedStats3 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats3.getStats().getStorageStats().containsKey((long) 10));
    // Write an empty HostAccountStorageStats
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    // Empty storage stats should remove all the data in the database
    HostAccountStorageStatsWrapper obtainedStats4 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats4.getStats().getStorageStats().isEmpty());
    // Write an empty HostAccountStorageStats again
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats3.getHeader(), new HostAccountStorageStats()));
    HostAccountStorageStatsWrapper obtainedStats5 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats5.getStats().getStorageStats().isEmpty());
    HostAccountStorageStatsWrapper stats6 = generateHostAccountStorageStatsWrapper(20, 20, 20, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats6);
    HostAccountStorageStatsWrapper obtainedStats6 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertEquals(obtainedStats6.getStats().getStorageStats(), stats6.getStats().getStorageStats());
    mySqlStore.shutdown();
}

Example 7 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStore method queryHostAccountStorageStatsBySimplifiedHostName.

/**
 * Query mysql database to get all the container storage usage for given {@code clusterName} and {@code queryHostname} and
 * construct a {@link HostAccountStorageStatsWrapper} from them
 * @param queryHostname The simplified hostname to query.
 * @return {@link HostAccountStorageStatsWrapper} published by the given simplified hostname.
 * @throws SQLException
 */
private HostAccountStorageStatsWrapper queryHostAccountStorageStatsBySimplifiedHostName(String queryHostname) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats();
    AtomicLong timestamp = new AtomicLong(0);
    accountReportsDao.queryStorageUsageForHost(clusterName, queryHostname, (partitionId, accountId, containerStats, updatedAtMs) -> {
        hostAccountStorageStats.addContainerStorageStats(partitionId, accountId, containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAtMs));
    });
    storeMetrics.queryStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return new HostAccountStorageStatsWrapper(new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp.get(), hostAccountStorageStats.getStorageStats().size(), hostAccountStorageStats.getStorageStats().size(), null), hostAccountStorageStats);
}

Example 8 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class AccountStatsMySqlStoreTest method testLocalBackupFile.

@Test
public void testLocalBackupFile() throws IOException {
    // First, make sure there is no local backup file.
    Path tempDir = Files.createTempDirectory("AccountStatsMySqlStoreTest");
    Path localBackupFilePath = tempDir.resolve("localbackup");
    Properties prop = new Properties();
    prop.setProperty(AccountStatsMySqlConfig.LOCAL_BACKUP_FILE_PATH, localBackupFilePath.toString());
    AccountStatsMySqlConfig accountStatsMySqlConfig = new AccountStatsMySqlConfig(new VerifiableProperties(prop));
    AccountStatsMySqlStore store = new AccountStatsMySqlStore(accountStatsMySqlConfig, mockDataSource, clusterName, hostname, null, new MetricRegistry());
    assertNull(store.getPreviousHostAccountStorageStatsWrapper());
    // Second, save a backup file.
    HostAccountStorageStats hostAccountStorageStats = new HostAccountStorageStats(StorageStatsUtilTest.generateRandomHostAccountStorageStats(10, 10, 10, 10000L, 2, 10));
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, System.currentTimeMillis(), 10, 10, null);
    HostAccountStorageStatsWrapper statsWrapper = new HostAccountStorageStatsWrapper(header, hostAccountStorageStats);
    ObjectMapper objectMapper = new ObjectMapper();
    objectMapper.writeValue(localBackupFilePath.toFile(), statsWrapper);
    store = new AccountStatsMySqlStore(accountStatsMySqlConfig, mockDataSource, clusterName, hostname, null, new MetricRegistry());
    HostAccountStorageStatsWrapper backupWrapper = store.getPreviousHostAccountStorageStatsWrapper();
    assertNotNull(backupWrapper);
    assertStatsHeader(backupWrapper.getHeader(), 10, 10);
    Assert.assertEquals(hostAccountStorageStats.getStorageStats(), backupWrapper.getStats().getStorageStats());
}

Example 9 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostAccountStorageStatsWrappers.

/**
 * Aggregate all {@link HostAccountStorageStatsWrapper} to generate two {@link AggregatedAccountStorageStats}s. First
 * {@link AggregatedAccountStorageStats} is the sum of all {@link HostAccountStorageStatsWrapper}s. The second {@link AggregatedAccountStorageStats}
 * is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostAccountStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedAccountStorageStats}.
 * @throws IOException
 */
Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregateHostAccountStorageStatsWrappers(Map<String, HostAccountStorageStatsWrapper> statsWrappers) throws IOException {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> combinedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> selectedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostAccountStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostAccountStorageStatsWrapper hostAccountStorageStatsWrapper = statsWrapperEntry.getValue();
        HostAccountStorageStats hostAccountStorageStats = hostAccountStorageStatsWrapper.getStats();
        HostAccountStorageStats hostAccountStorageStatsCopy1 = new HostAccountStorageStats(hostAccountStorageStats);
        HostAccountStorageStats hostAccountStorageStatsCopy2 = new HostAccountStorageStats(hostAccountStorageStats);
        combineRawHostAccountStorageStatsMap(combinedHostAccountStorageStatsMap, hostAccountStorageStatsCopy1.getStorageStats());
        selectRawHostAccountStorageStatsMap(selectedHostAccountStorageStatsMap, hostAccountStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostAccountStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostAccountStorageStats {}", mapper.writeValueAsString(combinedHostAccountStorageStatsMap));
        logger.trace("Selected raw HostAccountStorageStats {}", mapper.writeValueAsString(selectedHostAccountStorageStatsMap));
    }
    AggregatedAccountStorageStats combinedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(combinedHostAccountStorageStatsMap));
    AggregatedAccountStorageStats selectedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(selectedHostAccountStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}

Example 10 with HostAccountStorageStatsWrapper

use of com.github.ambry.server.HostAccountStorageStatsWrapper in project ambry by linkedin.

the class MySqlReportAggregatorTask method run.

@Override
public TaskResult run() {
    Exception exception = null;
    Histogram fetchTimeMs = statsReportType == StatsReportType.ACCOUNT_REPORT ? metrics.accountStatsFetchTimeMs : metrics.partitionClassStatsFetchTimeMs;
    Histogram aggregationTimeMs = statsReportType == StatsReportType.ACCOUNT_REPORT ? metrics.accountStatsAggregationTimeMs : metrics.partitionClassStatsAggregationTimeMs;
    long startTimeMs = System.currentTimeMillis();
    StatsSnapshot accountPhysicalStorageSnapshot = null;
    try {
        List<String> instanceNames = manager.getClusterManagmentTool().getInstancesInCluster(manager.getClusterName());
        if (statsReportType == StatsReportType.ACCOUNT_REPORT) {
            Map<String, HostAccountStorageStatsWrapper> accountStatsWrappers = fetchAccountStorageStatsWrapperForInstances(instanceNames);
            fetchTimeMs.update(System.currentTimeMillis() - startTimeMs);
            logger.info("Aggregating stats from " + accountStatsWrappers.size() + " hosts");
            Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> results = clusterAggregator.aggregateHostAccountStorageStatsWrappers(accountStatsWrappers);
            if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                removeInvalidAggregatedAccountAndContainerStats(results.getSecond());
            }
            accountStatsStore.storeAggregatedAccountStorageStats(results.getSecond());
            accountPhysicalStorageSnapshot = StorageStatsUtil.convertAggregatedAccountStorageStatsToStatsSnapshot(results.getFirst(), true);
        } else if (statsReportType == StatsReportType.PARTITION_CLASS_REPORT) {
            Map<String, HostPartitionClassStorageStatsWrapper> statsWrappers = fetchPartitionClassStorageStatsWrapperForInstances(instanceNames);
            fetchTimeMs.update(System.currentTimeMillis() - startTimeMs);
            logger.info("Aggregating stats from " + statsWrappers.size() + " hosts");
            Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> results = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(statsWrappers);
            if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                removeInvalidAggregatedPartitionClassStats(results.getSecond());
            }
            accountStatsStore.storeAggregatedPartitionClassStorageStats(results.getSecond());
        }
        // Check if there is a base report for this month or not.
        if (clusterMapConfig.clustermapEnableAggregatedMonthlyAccountReport && statsReportType == StatsReportType.ACCOUNT_REPORT) {
            // Get the month, if not the same month, then copy the aggregated stats and update the month
            String currentMonthValue = LocalDateTime.ofEpochSecond(time.seconds(), 0, ZONE_OFFSET).format(TIMESTAMP_FORMATTER);
            String recordedMonthValue = accountStatsStore.queryRecordedMonth();
            if (recordedMonthValue == null || recordedMonthValue.isEmpty() || !currentMonthValue.equals(recordedMonthValue)) {
                if (clusterMapConfig.clustermapEnableDeleteInvalidDataInMysqlAggregationTask) {
                    accountStatsStore.deleteSnapshotOfAggregatedAccountStats();
                }
                logger.info("Taking snapshot of aggregated stats for month " + currentMonthValue);
                accountStatsStore.takeSnapshotOfAggregatedAccountStatsAndUpdateMonth(currentMonthValue);
            }
        }
        aggregationTimeMs.update(System.currentTimeMillis() - startTimeMs);
        return new TaskResult(TaskResult.Status.COMPLETED, "Aggregation success");
    } catch (Exception e) {
        logger.error("Exception thrown while aggregating stats from container stats reports across all nodes ", e);
        exception = e;
        return new TaskResult(TaskResult.Status.FAILED, "Exception thrown");
    } finally {
        if (clusterMapConfig.clustermapEnableContainerDeletionAggregation && callback != null && accountPhysicalStorageSnapshot != null && statsReportType.equals(StatsReportType.ACCOUNT_REPORT)) {
            callback.onCompletion(accountPhysicalStorageSnapshot, exception);
        }
    }
}