Examples with StatsWrapper com.github.ambry.server.StatsWrapper used on opensource projects

Example 11 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregator method doWork.

/**
 * Take a {@link Map} of instance name to JSON string representation of {@link StatsWrapper} objects and perform cluster wide
 * aggregation with them.
 * @param statsWrappersJSON a {@link Map} of instance name to JSON string representation of {@link StatsWrapper} objects from the
 *                          node level
 * @return a {@link Pair} of Strings whose values represents valid quota stats across all partitions.
 * First element is the raw (sum) aggregated stats and second element is average(aggregated) stats for all replicas
 * for each partition.
 * @throws IOException
 */
Pair<String, String> doWork(Map<String, String> statsWrappersJSON) throws IOException {
    StatsSnapshot partitionSnapshot = new StatsSnapshot(0L, new HashMap<String, StatsSnapshot>());
    Map<String, Long> partitionTimestampMap = new HashMap<>();
    StatsSnapshot rawPartitionSnapshot = new StatsSnapshot(0L, new HashMap<String, StatsSnapshot>());
    for (Map.Entry<String, String> statsWrapperJSON : statsWrappersJSON.entrySet()) {
        if (statsWrapperJSON != null) {
            StatsWrapper snapshotWrapper = mapper.readValue(statsWrapperJSON.getValue(), StatsWrapper.class);
            StatsWrapper snapshotWrapperCopy = mapper.readValue(statsWrapperJSON.getValue(), StatsWrapper.class);
            combineRaw(rawPartitionSnapshot, snapshotWrapper);
            combine(partitionSnapshot, snapshotWrapperCopy, statsWrapperJSON.getKey(), partitionTimestampMap);
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw snapshot {}", mapper.writeValueAsString(rawPartitionSnapshot));
        logger.trace("Combined snapshot {}", mapper.writeValueAsString(partitionSnapshot));
    }
    StatsSnapshot reducedRawSnapshot = reduce(rawPartitionSnapshot);
    StatsSnapshot reducedSnapshot = reduce(partitionSnapshot);
    if (logger.isTraceEnabled()) {
        logger.trace("Reduced raw snapshot {}", mapper.writeValueAsString(reducedRawSnapshot));
        logger.trace("Reduced snapshot {}", mapper.writeValueAsString(reducedSnapshot));
    }
    return new Pair<>(mapper.writeValueAsString(reducedRawSnapshot), mapper.writeValueAsString(reducedSnapshot));
}

Example 12 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregator method doWork.

/**
 * Take a {@link Map} of instance name to JSON string representation of {@link StatsWrapper} objects and perform cluster wide
 * aggregation with them.
 * @param statsWrappersJSON a {@link Map} of instance name to JSON string representation of {@link StatsWrapper} objects from the
 *                          node level
 * @param type the type of stats report to be aggregated, which is defined in {@link StatsReportType}
 * @return a {@link Pair} of StatsSnapshot whose values represents aggregated stats across all partitions.
 * First element is the raw (sum) aggregated stats and second element is valid aggregated stats for all replicas
 * for each partition.
 * @throws IOException
 */
Pair<StatsSnapshot, StatsSnapshot> doWork(Map<String, String> statsWrappersJSON, StatsReportType type) throws IOException {
    Map<String, StatsWrapper> statsWrappers = new HashMap<>();
    for (Map.Entry<String, String> statsWrapperJSON : statsWrappersJSON.entrySet()) {
        try {
            if (statsWrapperJSON != null && statsWrapperJSON.getValue() != null) {
                StatsWrapper snapshotWrapper = mapper.readValue(statsWrapperJSON.getValue(), StatsWrapper.class);
                statsWrappers.put(statsWrapperJSON.getKey(), snapshotWrapper);
            }
        } catch (Exception e) {
            logger.error("Exception occurred while processing stats from {}", statsWrapperJSON.getKey(), e);
            exceptionOccurredInstances.computeIfAbsent(type, key -> new ArrayList<>()).add(statsWrapperJSON.getKey());
        }
    }
    return doWorkOnStatsWrapperMap(statsWrappers, type, false);
}

Example 13 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithDiffNumberOfStores.

/**
 * Test stats aggregation with different number of stores on different nodes.
 * Only used for partitionClass aggregation testing.
 * @throws IOException
 */
@Test
public void testDoWorkWithDiffNumberOfStores() throws IOException {
    List<StatsSnapshot> storeSnapshots1 = new ArrayList<>();
    List<StatsSnapshot> storeSnapshots2 = new ArrayList<>();
    List<StatsSnapshot> storeSnapshots2Copy = new ArrayList<>();
    int seed = 1111;
    // storeSnapshots1 only has 2 store stats. storeSnapshots2 and storeSnapshots2Copy have 3 store stats each.
    for (int i = 3; i < 6; i++) {
        if (i < 5) {
            storeSnapshots1.add(TestUtils.generateStoreStats(i, 3, new Random(seed), StatsReportType.PARTITION_CLASS_REPORT));
        }
        storeSnapshots2.add(TestUtils.generateStoreStats(i, 3, new Random(seed), StatsReportType.PARTITION_CLASS_REPORT));
        storeSnapshots2Copy.add(TestUtils.generateStoreStats(i, 3, new Random(seed), StatsReportType.PARTITION_CLASS_REPORT));
    }
    StatsWrapper nodeStatsWrapper1 = TestUtils.generateNodeStats(storeSnapshots1, DEFAULT_TIMESTAMP, StatsReportType.PARTITION_CLASS_REPORT);
    StatsWrapper nodeStatsWrapper2 = TestUtils.generateNodeStats(storeSnapshots2, DEFAULT_TIMESTAMP, StatsReportType.PARTITION_CLASS_REPORT);
    StatsWrapper nodeStatsWrapper2Copy = TestUtils.generateNodeStats(storeSnapshots2Copy, DEFAULT_TIMESTAMP, StatsReportType.PARTITION_CLASS_REPORT);
    Map<String, String> instanceStatsMap = new LinkedHashMap<>();
    instanceStatsMap.put("Instance_1", mapper.writeValueAsString(nodeStatsWrapper1));
    instanceStatsMap.put("Instance_2", mapper.writeValueAsString(nodeStatsWrapper2));
    Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceStatsMap, StatsReportType.PARTITION_CLASS_REPORT);
    // verify aggregation on raw data
    StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, null);
    StatsSnapshot.aggregate(expectedRawSnapshot, nodeStatsWrapper1.getSnapshot());
    StatsSnapshot.aggregate(expectedRawSnapshot, nodeStatsWrapper2Copy.getSnapshot());
    expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
    StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
    assertTrue("Mismatch in the raw data aggregated snapshot", expectedRawSnapshot.equals(rawSnapshot));
    // verify aggregation on valid data
    StatsSnapshot expectedValidsnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStatsWrapper2.getSnapshot());
    StatsSnapshot validSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
    assertTrue("Mismatch in the valid data aggregated snapshot", expectedValidsnapshot.equals(validSnapshot));
}

Example 14 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testStatsAggregationWithZeroValueSnapshots.

/**
 * Test removing zero value snapshots from aggregated result.
 * @throws IOException
 */
@Test
public void testStatsAggregationWithZeroValueSnapshots() throws IOException {
    int nodeCount = 3;
    Random random = new Random();
    // For each type of report, create snapshots for 3 stores with 3 accounts, 4 accounts and 5 accounts respectively.
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> storeSnapshots = new ArrayList<>();
        for (int i = 3; i < 6; i++) {
            storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
        }
        // add zero value to the first snapshot
        if (type == StatsReportType.ACCOUNT_REPORT) {
            Map<String, StatsSnapshot> accountStatsSnapshotMap = storeSnapshots.get(0).getSubMap();
            accountStatsSnapshotMap.put("A[100]", new StatsSnapshot(0L, new HashMap<String, StatsSnapshot>() {

                {
                    put("C[100]", new StatsSnapshot(0L, null));
                }
            }));
        } else {
            Map<String, StatsSnapshot> accountContainerStatsSnapshotMap = storeSnapshots.get(0).getSubMap();
            accountContainerStatsSnapshotMap.put("A[100]_C[100]", new StatsSnapshot(0L, null));
        }
        StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, DEFAULT_TIMESTAMP, type);
        String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, nodeStatsJSON);
        }
        // 1. Aggregate all snapshots into the first snapshot in snapshots list. The intention is to get expected aggregated snapshot.
        // 2. Then invoke clusterAggregator to do work on stats across all instances.
        // 3. Verify both raw stats and valid stats after aggregation
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        // Remove 0 values snapshots first
        StatsSnapshot expectedSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                storeSnapshots.get(0).getSubMap().remove("A[100]");
                // we expect for valid data aggregation.
                for (int i = 1; i < storeSnapshots.size(); i++) {
                    StatsSnapshot.aggregate(storeSnapshots.get(0), storeSnapshots.get(i));
                }
                expectedSnapshot = storeSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                storeSnapshots.get(0).getSubMap().remove("A[100]_C[100]");
                // Invoke reduceByPartitionClass to remove partition level and only keep the partition class and account_container entries
                expectedSnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStats.getSnapshot());
                break;
        }
        // Verify cluster wide raw stats aggregation
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertEquals("Mismatch in total value of " + type, nodeCount * expectedSnapshot.getValue(), rawSnapshot.getValue());
        if (type == StatsReportType.ACCOUNT_REPORT) {
            verifyAggregatedRawStatsForAccountReport(rawSnapshot, expectedSnapshot, nodeCount);
        } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            verifyAggregatedRawStatsForPartitionClassReport(rawSnapshot, expectedSnapshot, nodeCount);
        }
        // Verify cluster wide stats aggregation
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated snapshot", expectedSnapshot.equals(actualSnapshot));
    }
}

Example 15 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithOutdatedNode.

/**
 * Tests to verify cluster wide aggregation with outdated node stats.
 * @throws IOException
 */
@Test
public void testDoWorkWithOutdatedNode() throws IOException {
    long seed = 1111;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> upToDateStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> outdatedStoreSnapshots = new ArrayList<>();
        upToDateStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        outdatedStoreSnapshots.add(TestUtils.generateStoreStats(6, 3, new Random(seed), type));
        StatsWrapper upToDateNodeStats = TestUtils.generateNodeStats(upToDateStoreSnapshots, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), type);
        StatsWrapper outdatedNodeStats = TestUtils.generateNodeStats(outdatedStoreSnapshots, 0, type);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), type);
        Map<String, String> instanceToStatsMap = new LinkedHashMap<>();
        instanceToStatsMap.put("Instance_0", mapper.writeValueAsString(outdatedNodeStats));
        instanceToStatsMap.put("Instance_1", mapper.writeValueAsString(upToDateNodeStats));
        instanceToStatsMap.put("Instance_2", mapper.writeValueAsString(emptyNodeStats));
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedValidSnapshot = null;
        StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, new HashMap<>());
        switch(type) {
            case ACCOUNT_REPORT:
                expectedValidSnapshot = upToDateStoreSnapshots.get(0);
                StatsSnapshot.aggregate(expectedRawSnapshot, outdatedStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, upToDateStoreSnapshots.get(0));
                break;
            case PARTITION_CLASS_REPORT:
                expectedValidSnapshot = HelixClusterAggregator.reduceByPartitionClass(upToDateNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, outdatedNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, upToDateNodeStats.getSnapshot());
                expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
                break;
        }
        // verify cluster wide aggregation on raw stats with outdated node stats
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated raw snapshot", expectedRawSnapshot.equals(rawSnapshot));
        // verify cluster wide aggregation on valid stats with outdated node stats
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated valid snapshot", expectedValidSnapshot.equals(actualSnapshot));
        // verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}