Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 21 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class BlobStoreStats method getContainerStorageStats.

/**
 * Gets the storage stats for all serviceIds and their containerIds as of now (the time when the API is called).
 * Storage stats is comprised of 3 values: 1. valid data size (logicalStorageUsage) 2. physical data size 3. number of blobs.
 * The following data are considered as valid data for this API:
 * 1. PUT with no expiry and no corresponding DELETE
 * 2. PUT expiring at t_exp but t_exp_ref < t_exp
 * 3. PUT with corresponding DELETE at time t_delete but t_del_ref < t_delete
 * For this API, t_ref is specified by the given reference time.
 * For physical data size, all the records will be taken into consideration, including DELETED PUT, even DELETE record itself.
 * For number of blobs, it includes all different blob ids.
 * @param referenceTimeInMs the reference time in ms until which deletes and expiration are relevant
 * @return the storage stats of each container in the form of a nested {@link Map} of accountId to another map of containerId
 * to {@link ContainerStorageStats}.
 */
Map<Short, Map<Short, ContainerStorageStats>> getContainerStorageStats(long referenceTimeInMs) throws StoreException {
    if (!enabled.get()) {
        throw new StoreException(String.format("BlobStoreStats is not enabled or closing for store %s", storeId), StoreErrorCodes.Store_Shutting_Down);
    }
    Map<Short, Map<Short, Long>> validSizeMap = null;
    Map<Short, Map<Short, Long>> physicalUsageMap = null;
    Map<Short, Map<Short, Long>> numberStoreKeyMap = null;
    ScanResults currentScanResults = scanResults.get();
    if (currentScanResults != null && isWithinRange(currentScanResults.containerForecastStartTimeMs, currentScanResults.containerForecastEndTimeMs, referenceTimeInMs)) {
        validSizeMap = currentScanResults.getValidSizePerContainer(referenceTimeInMs);
        physicalUsageMap = currentScanResults.getContainerPhysicalStorageUsage();
        numberStoreKeyMap = currentScanResults.getContainerNumberOfStoreKeys();
    } else {
        if (isScanning && isWithinRange(indexScanner.newScanResults.containerForecastStartTimeMs, indexScanner.newScanResults.containerForecastEndTimeMs, referenceTimeInMs)) {
            scanLock.lock();
            try {
                if (isScanning) {
                    if (waitCondition.await(waitTimeoutInSecs, TimeUnit.SECONDS)) {
                        currentScanResults = scanResults.get();
                        if (isWithinRange(currentScanResults.containerForecastStartTimeMs, currentScanResults.containerForecastEndTimeMs, referenceTimeInMs)) {
                            validSizeMap = currentScanResults.getValidSizePerContainer(referenceTimeInMs);
                            physicalUsageMap = currentScanResults.getContainerPhysicalStorageUsage();
                            numberStoreKeyMap = currentScanResults.getContainerNumberOfStoreKeys();
                        }
                    } else {
                        metrics.blobStoreStatsIndexScannerErrorCount.inc();
                        logger.error("Timed out while waiting for BlobStoreStats index scan to complete for store {}", storeId);
                    }
                } else {
                    currentScanResults = scanResults.get();
                    if (isWithinRange(currentScanResults.containerForecastStartTimeMs, currentScanResults.containerForecastEndTimeMs, referenceTimeInMs)) {
                        validSizeMap = currentScanResults.getValidSizePerContainer(referenceTimeInMs);
                        physicalUsageMap = currentScanResults.getContainerPhysicalStorageUsage();
                        numberStoreKeyMap = currentScanResults.getContainerNumberOfStoreKeys();
                    }
                }
            } catch (InterruptedException e) {
                metrics.blobStoreStatsIndexScannerErrorCount.inc();
                throw new IllegalStateException(String.format("Illegal state, wait for scan to complete is interrupted for store %s", storeId), e);
            } finally {
                scanLock.unlock();
            }
        }
        if (validSizeMap == null) {
            // 3. rare edge case where currentScanResults updated twice since the start of the wait.
            return collectContainerStorageStats(referenceTimeInMs);
        }
    }
    Map<Short, Map<Short, ContainerStorageStats>> retValue = new HashMap<>();
    for (short accountId : validSizeMap.keySet()) {
        for (short containerId : validSizeMap.get(accountId).keySet()) {
            retValue.computeIfAbsent(accountId, k -> new HashMap<>()).put(containerId, new ContainerStorageStats(containerId, validSizeMap.get(accountId).get(containerId), physicalUsageMap.get(accountId).get(containerId), numberStoreKeyMap.get(accountId).get(containerId)));
        }
    }
    return retValue;
}

Example 22 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class BlobStoreStats method getContainerStorageStats.

@Override
public Map<Short, Map<Short, ContainerStorageStats>> getContainerStorageStats(long referenceTimeInMs, List<Short> accountIdsToExclude) throws StoreException {
    Map<Short, Map<Short, ContainerStorageStats>> containerStatsMap = getContainerStorageStats(referenceTimeInMs);
    if (accountIdsToExclude != null && !accountIdsToExclude.isEmpty()) {
        accountIdsToExclude.forEach(id -> containerStatsMap.remove(id));
    }
    // Remove zero storage stats
    List<Short> accountIdToRemove = new ArrayList<>();
    List<Short> containerIdToRemove = new ArrayList<>();
    for (short accountId : containerStatsMap.keySet()) {
        containerIdToRemove.clear();
        for (short containerId : containerStatsMap.get(accountId).keySet()) {
            ContainerStorageStats stats = containerStatsMap.get(accountId).get(containerId);
            if (stats.isEmpty()) {
                containerIdToRemove.add(containerId);
            }
        }
        for (short containerId : containerIdToRemove) {
            containerStatsMap.get(accountId).remove(containerId);
        }
        if (containerStatsMap.get(accountId).size() == 0) {
            accountIdToRemove.add(accountId);
        }
    }
    for (short accountId : accountIdToRemove) {
        containerStatsMap.remove(accountId);
    }
    return containerStatsMap;
}

Example 23 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class BlobStoreStats method collectContainerStorageStats.

/**
 * Walk through the entire index and collect storage stats per container.
 * @param referenceTimeInMs the reference time in ms until which deletes and expiration are relevant
 * @return a nested {@link Map} of serviceId to containerId to {@link ContainerStorageStats}.
 */
private Map<Short, Map<Short, ContainerStorageStats>> collectContainerStorageStats(long referenceTimeInMs) throws StoreException {
    logger.trace("On demand index scanning to collect container valid data sizes for store {} wrt ref time {}", storeId, referenceTimeInMs);
    long startTimeMs = time.milliseconds();
    Map<StoreKey, IndexFinalState> keyFinalStates = new HashMap<>();
    Map<Short, Map<Short, Long>> validDataSizePerContainer = new HashMap<>();
    Map<Short, Map<Short, Long>> physicalDataSizePerContainer = new HashMap<>();
    Map<Short, Map<Short, Long>> storeKeysPerContainer = new HashMap<>();
    Map<Short, Map<Short, ContainerStorageStats>> result = new HashMap<>();
    int indexSegmentCount = 0;
    for (IndexSegment indexSegment : index.getIndexSegments().descendingMap().values()) {
        if (!enabled.get()) {
            throw new StoreException(String.format("BlobStoreStats is not enabled or closing for store %s", storeId), StoreErrorCodes.Store_Shutting_Down);
        }
        long indexSegmentStartProcessTimeMs = time.milliseconds();
        diskIOScheduler.getSlice(BlobStoreStats.IO_SCHEDULER_JOB_TYPE, BlobStoreStats.IO_SCHEDULER_JOB_ID, indexSegment.size());
        forEachIndexEntry(indexSegment, referenceTimeInMs, time.milliseconds(), null, keyFinalStates, true, (entry, isValid) -> {
            IndexValue indexValue = entry.getValue();
            if (isValid && indexValue.isPut()) {
                // delete and TTL update records does not count towards valid data size for usage (containers)
                updateNestedMapHelper(validDataSizePerContainer, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            }
            updateNestedMapHelper(physicalDataSizePerContainer, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            updateNestedMapHelper(storeKeysPerContainer, indexValue.getAccountId(), indexValue.getContainerId(), (long) (indexValue.isPut() ? 1 : 0));
        });
        metrics.statsOnDemandScanTimePerIndexSegmentMs.update(time.milliseconds() - indexSegmentStartProcessTimeMs, TimeUnit.MILLISECONDS);
        indexSegmentCount++;
        if (indexSegmentCount == 1 || indexSegmentCount % 10 == 0) {
            logger.info("Container Stats: Index segment {} processing complete (on-demand scanning) for store {}", indexSegment.getFile().getName(), storeId);
        }
    }
    for (short accountId : validDataSizePerContainer.keySet()) {
        for (short containerId : validDataSizePerContainer.get(accountId).keySet()) {
            result.computeIfAbsent(accountId, k -> new HashMap<>()).put(containerId, new ContainerStorageStats(containerId, validDataSizePerContainer.get(accountId).get(containerId), physicalDataSizePerContainer.get(accountId).get(containerId), storeKeysPerContainer.get(accountId).get(containerId)));
        }
    }
    // The remaining index entries in keyFinalStates are DELETE tombstones left by compaction (whose associated PUT is not found)
    updateDeleteTombstoneStats(keyFinalStates.values());
    metrics.statsOnDemandScanTotalTimeMs.update(time.milliseconds() - startTimeMs, TimeUnit.MILLISECONDS);
    return result;
}

Example 24 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StatsManagerTest method testStatsManagerDeleteTombstoneStats.

/**
 * Test to verify that the {@link StatsManager} is collecting delete tombstone stats.
 */
@Test
public void testStatsManagerDeleteTombstoneStats() {
    List<PartitionId> unreachablePartitions = Collections.emptyList();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> hostAccountStorageStatsMap = new HashMap<>();
    for (PartitionId partitionId : storeMap.keySet()) {
        statsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
    }
    statsManager.updateAggregatedDeleteTombstoneStats();
    // verify aggregated delete tombstone stats
    StatsManager.AggregatedDeleteTombstoneStats deleteTombstoneStats = statsManager.getAggregatedDeleteTombstoneStats();
    Pair<Long, Long> expectedExpiredDeleteStats = storeDeleteTombstoneStats.get(EXPIRED_DELETE_TOMBSTONE);
    Pair<Long, Long> expectedPermanentDeleteStats = storeDeleteTombstoneStats.get(PERMANENT_DELETE_TOMBSTONE);
    assertEquals("Mismatch in expired delete count", storeMap.size() * expectedExpiredDeleteStats.getFirst(), deleteTombstoneStats.getExpiredDeleteTombstoneCount());
    assertEquals("Mismatch in expired delete size", storeMap.size() * expectedExpiredDeleteStats.getSecond(), deleteTombstoneStats.getExpiredDeleteTombstoneSize());
    assertEquals("Mismatch in permanent delete count", storeMap.size() * expectedPermanentDeleteStats.getFirst(), deleteTombstoneStats.getPermanentDeleteTombstoneCount());
    assertEquals("Mismatch in permanent delete size", storeMap.size() * expectedPermanentDeleteStats.getSecond(), deleteTombstoneStats.getPermanentDeleteTombstoneSize());
}

Example 25 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostAccountStorageStatsWrappers.

/**
 * Aggregate all {@link HostAccountStorageStatsWrapper} to generate two {@link AggregatedAccountStorageStats}s. First
 * {@link AggregatedAccountStorageStats} is the sum of all {@link HostAccountStorageStatsWrapper}s. The second {@link AggregatedAccountStorageStats}
 * is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostAccountStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedAccountStorageStats}.
 * @throws IOException
 */
Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregateHostAccountStorageStatsWrappers(Map<String, HostAccountStorageStatsWrapper> statsWrappers) throws IOException {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> combinedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> selectedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostAccountStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostAccountStorageStatsWrapper hostAccountStorageStatsWrapper = statsWrapperEntry.getValue();
        HostAccountStorageStats hostAccountStorageStats = hostAccountStorageStatsWrapper.getStats();
        HostAccountStorageStats hostAccountStorageStatsCopy1 = new HostAccountStorageStats(hostAccountStorageStats);
        HostAccountStorageStats hostAccountStorageStatsCopy2 = new HostAccountStorageStats(hostAccountStorageStats);
        combineRawHostAccountStorageStatsMap(combinedHostAccountStorageStatsMap, hostAccountStorageStatsCopy1.getStorageStats());
        selectRawHostAccountStorageStatsMap(selectedHostAccountStorageStatsMap, hostAccountStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostAccountStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostAccountStorageStats {}", mapper.writeValueAsString(combinedHostAccountStorageStatsMap));
        logger.trace("Selected raw HostAccountStorageStats {}", mapper.writeValueAsString(selectedHostAccountStorageStatsMap));
    }
    AggregatedAccountStorageStats combinedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(combinedHostAccountStorageStatsMap));
    AggregatedAccountStorageStats selectedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(selectedHostAccountStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}