Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 16 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method storeHostAccountStorageStats.

/**
 * Store the {@link HostAccountStorageStatsWrapper} to mysql database. This method ignores the error information from {@link HostAccountStorageStatsWrapper}
 * and only publish the container storage usages that are different from the previous one.
 * @param statsWrapper The {@link HostAccountStorageStatsWrapper} to publish.
 */
@Override
public void storeHostAccountStorageStats(HostAccountStorageStatsWrapper statsWrapper) throws Exception {
    AccountReportsDao.StorageBatchUpdater batch = accountReportsDao.new StorageBatchUpdater(config.updateBatchSize);
    int batchSize = 0;
    long startTimeMs = System.currentTimeMillis();
    HostAccountStorageStats prevHostStats = Optional.ofNullable(previousHostAccountStorageStatsWrapper).map(HostAccountStorageStatsWrapper::getStats).orElseGet(HostAccountStorageStats::new);
    HostAccountStorageStats currHostStats = statsWrapper.getStats();
    // Find the differences between two {@link HostAccountStorageStats} and apply them to the given {@link ContainerStorageStatsFunction}.
    // The difference is defined as
    // 1. If a container storage usage exists in both HostAccountStorageStats, and the values are different.
    // 2. If a container storage usage only exists in current HostAccountStorageStats.
    // If a container storage usage only exists in the previous HostAccountStorageStats, then it will not be applied to the given function.
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> currPartitionMap = currHostStats.getStorageStats();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> prevPartitionMap = prevHostStats.getStorageStats();
    for (Map.Entry<Long, Map<Short, Map<Short, ContainerStorageStats>>> currPartitionMapEntry : currPartitionMap.entrySet()) {
        long partitionId = currPartitionMapEntry.getKey();
        Map<Short, Map<Short, ContainerStorageStats>> currAccountMap = currPartitionMapEntry.getValue();
        Map<Short, Map<Short, ContainerStorageStats>> prevAccountMap = prevPartitionMap.getOrDefault(partitionId, Collections.emptyMap());
        // as its value.
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> currAccountMapEntry : currAccountMap.entrySet()) {
            short accountId = currAccountMapEntry.getKey();
            Map<Short, ContainerStorageStats> currContainerMap = currAccountMapEntry.getValue();
            Map<Short, ContainerStorageStats> prevContainerMap = prevAccountMap.getOrDefault(accountId, Collections.emptyMap());
            for (Map.Entry<Short, ContainerStorageStats> currContainerMapEntry : currContainerMap.entrySet()) {
                short containerId = currContainerMapEntry.getKey();
                ContainerStorageStats currContainerStorageStats = currContainerMapEntry.getValue();
                if (!prevContainerMap.containsKey(containerId) || !currContainerStorageStats.equals(prevContainerMap.get(containerId))) {
                    batch.addUpdateToBatch(clusterName, hostname, (int) partitionId, accountId, currContainerStorageStats);
                    batchSize++;
                }
            }
        }
    }
    batch.flush();
    storeMetrics.batchSize.update(batchSize);
    storeMetrics.insertAccountStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    deleteContainerAccountStats(prevPartitionMap, currPartitionMap);
    storeMetrics.publishTimeMs.update(System.currentTimeMillis() - startTimeMs);
    previousHostAccountStorageStatsWrapper = statsWrapper;
    writeStatsToLocalBackupFile();
}

Example 17 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method storeAggregatedPartitionClassStorageStats.

@Override
public void storeAggregatedPartitionClassStorageStats(AggregatedPartitionClassStorageStats aggregatedPartitionClassStorageStats) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    PartitionClassReportsDao.StorageBatchUpdater batch = partitionClassReportsDao.new StorageBatchUpdater(config.updateBatchSize);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> storageStats = aggregatedPartitionClassStorageStats.getStorageStats();
    for (String partitionClassName : storageStats.keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> accountStorageStats = storageStats.get(partitionClassName);
        for (short accountId : accountStorageStats.keySet()) {
            for (ContainerStorageStats containerStats : accountStorageStats.get(accountId).values()) {
                batch.addUpdateToBatch(clusterName, partitionClassName, accountId, containerStats);
            }
        }
    }
    batch.flush();
    storeMetrics.storeAggregatedPartitionClassStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
}

Example 18 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AggregatedAccountReportsDao method queryContainerUsageForClusterInternal.

/**
 * Query container storage for the given {@code clusterName}. The result will be applied to the {@link AggregatedContainerStorageStatsFunction}.
 * @param forMonthly True to return the monthly snapshot of the container storage usage.
 * @param clusterName The clusterName.
 * @param func The {@link AggregatedContainerStorageStatsFunction} to call to process each container storage usage.
 * @throws SQLException
 */
private void queryContainerUsageForClusterInternal(boolean forMonthly, String clusterName, AggregatedContainerStorageStatsFunction func) throws SQLException {
    try (Connection connection = dataSource.getConnection()) {
        String sqlStatement = forMonthly ? queryMonthlyUsageSqlForCluster : queryUsageSqlForCluster;
        try (PreparedStatement queryStatement = connection.prepareStatement(sqlStatement)) {
            long startTimeMs = System.currentTimeMillis();
            queryStatement.setString(1, clusterName);
            try (ResultSet resultSet = queryStatement.executeQuery()) {
                while (resultSet.next()) {
                    int accountId = resultSet.getInt(ACCOUNT_ID_COLUMN);
                    int containerId = resultSet.getInt(CONTAINER_ID_COLUMN);
                    long storageUsage = resultSet.getLong(STORAGE_USAGE_COLUMN);
                    final long physicalStorageUsage = resultSet.getLong(PHYSICAL_STORAGE_USAGE_COLUMN);
                    final long numberOfBlobs = resultSet.getLong(NUMBER_OF_BLOBS_COLUMN);
                    func.apply((short) accountId, new ContainerStorageStats((short) containerId, storageUsage, physicalStorageUsage, numberOfBlobs));
                }
            }
            metrics.readTimeMs.update(System.currentTimeMillis() - startTimeMs);
            metrics.readSuccessCount.inc();
        }
    } catch (SQLException e) {
        metrics.readFailureCount.inc();
        logger.error(String.format("Failed to execute query on %s, with parameter %s", AGGREGATED_ACCOUNT_REPORTS_TABLE, clusterName), e);
        throw e;
    }
}

Example 19 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class PartitionClassReportsDao method queryAggregatedPartitionClassReport.

/**
 * Query container storage usage for given {@code clusterName}. This usage is aggregated under each partition class name.
 * The results will be applied to the given {@link PartitionClassContainerStorageStatsFunction}.
 * @param clusterName The clusterName.
 * @param func The callback to apply query results
 * @throws SQLException
 */
void queryAggregatedPartitionClassReport(String clusterName, PartitionClassContainerStorageStatsFunction func) throws SQLException {
    try (Connection connection = dataSource.getConnection()) {
        try (PreparedStatement queryStatement = connection.prepareStatement(queryAggregatedSql)) {
            long startTimeMs = System.currentTimeMillis();
            queryStatement.setString(1, clusterName);
            try (ResultSet rs = queryStatement.executeQuery()) {
                while (rs.next()) {
                    String partitionClassName = rs.getString(NAME_COLUMN);
                    int accountId = rs.getInt(ACCOUNT_ID_COLUMN);
                    int containerId = rs.getInt(CONTAINER_ID_COLUMN);
                    long usage = rs.getLong(STORAGE_USAGE_COLUMN);
                    final long physicalStorageUsage = rs.getLong(PHYSICAL_STORAGE_USAGE_COLUMN);
                    final long numberOfBlobs = rs.getLong(NUMBER_OF_BLOBS_COLUMN);
                    long updatedAt = rs.getTimestamp(UPDATED_AT_COLUMN).getTime();
                    func.apply(partitionClassName, (short) accountId, new ContainerStorageStats((short) containerId, usage, physicalStorageUsage, numberOfBlobs), updatedAt);
                }
            }
            metrics.readTimeMs.update(System.currentTimeMillis() - startTimeMs);
            metrics.readSuccessCount.inc();
        }
    } catch (SQLException e) {
        metrics.readFailureCount.inc();
        logger.error("Failed to execute query {}, with parameter {}", queryAggregatedSql, clusterName, e);
        throw e;
    }
}

Example 20 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StatsManager method collectAndAggregatePartitionClassStorageStats.

/**
 * Fetch and aggregate partition class stats from a given {@link Store}
 * @param hostPartitionClassStorageStatsMap map from partition class to all partition storage stats.
 * @param partitionId specifies the {@link Store} to be fetched from
 * @param unreachablePartitions a {@link List} containing partition Ids that were unable to successfully fetch from
 */
void collectAndAggregatePartitionClassStorageStats(Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> hostPartitionClassStorageStatsMap, PartitionId partitionId, List<PartitionId> unreachablePartitions) {
    Store store = storageManager.getStore(partitionId, false);
    if (store == null) {
        unreachablePartitions.add(partitionId);
    } else {
        try {
            long fetchAndAggregatePerStoreStartTimeMs = time.milliseconds();
            StoreStats storeStats = store.getStoreStats();
            Map<Short, Map<Short, ContainerStorageStats>> containerStatsMap = storeStats.getContainerStorageStats(time.milliseconds(), publishExcludeAccountIds);
            String partitionClassName = partitionId.getPartitionClass();
            hostPartitionClassStorageStatsMap.computeIfAbsent(partitionClassName, k -> new HashMap<>()).put(partitionId.getId(), containerStatsMap);
            metrics.fetchAndAggregateTimePerStoreMs.update(time.milliseconds() - fetchAndAggregatePerStoreStartTimeMs);
        } catch (StoreException e) {
            unreachablePartitions.add(partitionId);
        }
    }
}