Examples with ContainerStorageStats com.github.ambry.server.storagestats.ContainerStorageStats used on opensource projects

Example 36 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testEmptyStatsWhenReadingPreviousStatsFromMysqlDb.

@Test
public void testEmptyStatsWhenReadingPreviousStatsFromMysqlDb() throws Exception {
    // write a new stats into database.
    AccountStatsMySqlStore mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    HostAccountStorageStatsWrapper stats = generateHostAccountStorageStatsWrapper(1, 1, 1, StatsReportType.ACCOUNT_REPORT);
    mySqlStore.storeHostAccountStorageStats(stats);
    HostAccountStorageStatsWrapper obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertTrue(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
    // initialized the mySqlStore and write a new stats with the same partition.
    mySqlStore = createAccountStatsMySqlStore(clusterName1, hostname1, port1);
    assertTrue(mySqlStore.getPreviousHostAccountStorageStatsWrapper().getStats().getStorageStats().containsKey((long) 0));
    HostAccountStorageStatsWrapper stats2 = generateHostAccountStorageStatsWrapper(0, 0, 0, StatsReportType.ACCOUNT_REPORT);
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> newStorageStats = new HashMap<>(stats2.getStats().getStorageStats());
    newStorageStats.put((long) 0, // Remove partition 0's storage stats data, this would remove entire partition from database
    new HashMap<>());
    mySqlStore.storeHostAccountStorageStats(new HostAccountStorageStatsWrapper(stats2.getHeader(), new HostAccountStorageStats(newStorageStats)));
    // empty stats should remove all the data in the database
    obtainedStats = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    assertFalse(obtainedStats.getStats().getStorageStats().containsKey((long) 0));
}

Example 37 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountReportsDao method queryStorageUsageForHost.

/**
 * Query container storage usage for given {@code clusterName} and {@code hostname}. The result will be applied to the
 * {@link ContainerStorageStatsFunction}.
 * @param clusterName The clusterName.
 * @param hostname The hostname.
 * @param func The {@link ContainerStorageStatsFunction} to call to process each container storage usage.
 * @throws SQLException
 */
void queryStorageUsageForHost(String clusterName, String hostname, ContainerStorageStatsFunction func) throws SQLException {
    try (Connection connection = dataSource.getConnection()) {
        try (PreparedStatement queryStatement = connection.prepareStatement(querySqlForClusterAndHost)) {
            long startTimeMs = System.currentTimeMillis();
            queryStatement.setString(1, clusterName);
            queryStatement.setString(2, hostname);
            try (ResultSet resultSet = queryStatement.executeQuery()) {
                while (resultSet.next()) {
                    int partitionId = resultSet.getInt(PARTITION_ID_COLUMN);
                    int accountId = resultSet.getInt(ACCOUNT_ID_COLUMN);
                    int containerId = resultSet.getInt(CONTAINER_ID_COLUMN);
                    long storageUsage = resultSet.getLong(STORAGE_USAGE_COLUMN);
                    final long physicalStorageUsage = resultSet.getLong(PHYSICAL_STORAGE_USAGE_COLUMN);
                    final long numberOfBlobs = resultSet.getLong(NUMBER_OF_BLOBS_COLUMN);
                    long updatedAtMs = resultSet.getTimestamp(UPDATED_AT_COLUMN).getTime();
                    func.apply(partitionId, (short) accountId, new ContainerStorageStats((short) containerId, storageUsage, physicalStorageUsage, numberOfBlobs), updatedAtMs);
                }
            }
            metrics.readTimeMs.update(System.currentTimeMillis() - startTimeMs);
            metrics.readSuccessCount.inc();
        }
    } catch (SQLException e) {
        metrics.readFailureCount.inc();
        logger.error(String.format("Failed to execute query on %s, with parameter %s %s", ACCOUNT_REPORTS_TABLE, clusterName, hostname), e);
        throw e;
    }
}

Example 38 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class AccountStatsMySqlStore method queryHostPartitionClassStorageStatsByHost.

@Override
public HostPartitionClassStorageStatsWrapper queryHostPartitionClassStorageStatsByHost(String hostname, int port, Map<String, Set<Integer>> partitionNameAndIds) throws SQLException {
    long startTimeMs = System.currentTimeMillis();
    hostname = hostnameHelper.simplifyHostname(hostname, port);
    Map<Integer, Map<Short, Map<Short, ContainerStorageStats>>> partitionAccountContainerUsage = new HashMap<>();
    AtomicLong timestamp = new AtomicLong(0);
    accountReportsDao.queryStorageUsageForHost(clusterName, hostname, (partitionId, accountId, containerStats, updatedAtMs) -> {
        partitionAccountContainerUsage.computeIfAbsent(partitionId, pid -> new HashMap<>()).computeIfAbsent(accountId, aid -> new HashMap<>()).put(containerStats.getContainerId(), containerStats);
        timestamp.set(Math.max(timestamp.get(), updatedAtMs));
    });
    // Here partitionAccountContainerUsage has partition id, account id and container id as keys of map at each level,
    // the value is the storage usage.
    // As indicated by the comments above, we have to know the partition class name for each partition id. Luckily, we
    // have all the partition ids and we have a map partitionNameAndIds whose key is the partition class name and the
    // value is the list of all partition ids belong to the partition class name.
    Set<Integer> partitionIds = partitionAccountContainerUsage.keySet();
    Map<String, Set<Integer>> partitionNameAndIdsForHost = new HashMap<>();
    for (int partitionId : partitionIds) {
        boolean found = false;
        for (Map.Entry<String, Set<Integer>> namesAndIdsEntry : partitionNameAndIds.entrySet()) {
            if (namesAndIdsEntry.getValue().contains(partitionId)) {
                partitionNameAndIdsForHost.computeIfAbsent(namesAndIdsEntry.getKey(), k -> new HashSet<>()).add(partitionId);
                found = true;
                break;
            }
        }
        if (!found) {
            storeMetrics.missingPartitionClassNameErrorCount.inc();
            logger.error("Can't find partition class name for partition id {}", partitionId);
        }
    }
    HostPartitionClassStorageStats hostPartitionClassStorageStats = new HostPartitionClassStorageStats();
    for (Map.Entry<String, Set<Integer>> nameAndIdsEntry : partitionNameAndIdsForHost.entrySet()) {
        String partitionClassName = nameAndIdsEntry.getKey();
        for (int partitionId : nameAndIdsEntry.getValue()) {
            Map<Short, Map<Short, ContainerStorageStats>> accountContainerUsage = partitionAccountContainerUsage.get(partitionId);
            for (short accountId : accountContainerUsage.keySet()) {
                Map<Short, ContainerStorageStats> containerUsage = accountContainerUsage.get(accountId);
                containerUsage.values().stream().forEach(containerStats -> hostPartitionClassStorageStats.addContainerStorageStats(partitionClassName, partitionId, accountId, containerStats));
            }
        }
    }
    storeMetrics.queryPartitionClassStatsTimeMs.update(System.currentTimeMillis() - startTimeMs);
    return new HostPartitionClassStorageStatsWrapper(new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp.get(), partitionAccountContainerUsage.size(), partitionAccountContainerUsage.size(), null), hostPartitionClassStorageStats);
}

Example 39 with ContainerStorageStats

use of com.github.ambry.server.storagestats.ContainerStorageStats in project ambry by linkedin.

the class StatsManagerTest method testStatsManagerWithProblematicStores.

/**
 * Test to verify the behavior when dealing with {@link Store} that is null and when {@link StoreException} is thrown.
 * @throws Exception
 */
@Test
public void testStatsManagerWithProblematicStores() throws Exception {
    DataNodeId dataNodeId = new MockDataNodeId(Collections.singletonList(new Port(6667, PortType.PLAINTEXT)), Collections.singletonList("/tmp"), "DC1");
    Map<PartitionId, Store> problematicStoreMap = new HashMap<>();
    PartitionId partitionId1 = new MockPartitionId(1, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
    PartitionId partitionId2 = new MockPartitionId(2, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
    problematicStoreMap.put(partitionId1, null);
    Store exceptionStore = new MockStore(new MockStoreStats(new HashMap<>(), true));
    problematicStoreMap.put(partitionId2, exceptionStore);
    StatsManager testStatsManager = new StatsManager(new MockStorageManager(problematicStoreMap, dataNodeId), Arrays.asList(partitionId1.getReplicaIds().get(0), partitionId2.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
    List<PartitionId> unreachablePartitions = new ArrayList<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> hostAccountStorageStatsMap = new HashMap<>();
    for (PartitionId partitionId : problematicStoreMap.keySet()) {
        testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
    }
    assertEquals("Aggregated map should not contain any value", 0L, hostAccountStorageStatsMap.size());
    assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
    StatsManager.AccountStatsPublisher publisher = testStatsManager.new AccountStatsPublisher(accountStatsStore);
    publisher.run();
    HostAccountStorageStatsWrapper statsWrapper = accountStatsStore.queryHostAccountStorageStatsByHost("localhost", 0);
    List<String> unreachableStores = statsWrapper.getHeader().getUnreachableStores();
    assertTrue("The unreachable store list should contain Partition1 and Partition2", unreachableStores.containsAll(Arrays.asList(partitionId1.toPathString(), partitionId2.toPathString())));
    // test for the scenario where some stores are healthy and some are bad
    Map<PartitionId, Store> mixedStoreMap = new HashMap<>(storeMap);
    unreachablePartitions.clear();
    PartitionId partitionId3 = new MockPartitionId(3, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
    PartitionId partitionId4 = new MockPartitionId(4, MockClusterMap.DEFAULT_PARTITION_CLASS, Collections.singletonList((MockDataNodeId) dataNodeId), 0);
    mixedStoreMap.put(partitionId3, null);
    mixedStoreMap.put(partitionId4, exceptionStore);
    testStatsManager = new StatsManager(new MockStorageManager(mixedStoreMap, dataNodeId), Arrays.asList(partitionId3.getReplicaIds().get(0), partitionId4.getReplicaIds().get(0)), new MetricRegistry(), statsManagerConfig, new MockTime(), null, null, inMemoryAccountService);
    hostAccountStorageStatsMap.clear();
    for (PartitionId partitionId : mixedStoreMap.keySet()) {
        testStatsManager.collectAndAggregateAccountStorageStats(hostAccountStorageStatsMap, partitionId, unreachablePartitions);
    }
    assertEquals("Unreachable store count mismatch with expected value", 2, unreachablePartitions.size());
    // test fetchSnapshot method in StatsManager
    unreachablePartitions.clear();
    // partition 0, 1, 2 are healthy stores, partition 3, 4 are bad ones.
    for (PartitionId partitionId : mixedStoreMap.keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> containerStatsMapForPartition = hostAccountStorageStatsMap.get(partitionId.getId());
        if (partitionId.getId() < 3) {
            assertEquals("Actual map does not match with expected snapshot with partition id " + partitionId.toPathString(), hostAccountStorageStats.getStorageStats().get(partitionId.getId()), containerStatsMapForPartition);
        }
    }
}