Examples with StatsWrapper com.github.ambry.server.StatsWrapper used on opensource projects

Example 1 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method generateNodeStats.

/**
 * Given a {@link List} of {@link StatsSnapshot}s and a timestamp generate a {@link StatsWrapper} that would have been
 * produced by a node.
 * @param storeSnapshots a {@link List} of store level {@link StatsSnapshot}s.
 * @param timestamp the timestamp to be attached to the generated {@link StatsWrapper}
 * @return the generated node level {@link StatsWrapper}
 */
private StatsWrapper generateNodeStats(List<StatsSnapshot> storeSnapshots, long timestamp) {
    Map<String, StatsSnapshot> partitionMap = new HashMap<>();
    long total = 0;
    int numbOfPartitions = storeSnapshots.size();
    for (int i = 0; i < numbOfPartitions; i++) {
        StatsSnapshot partitionSnapshot = storeSnapshots.get(i);
        partitionMap.put(String.format("partition_%d", i), partitionSnapshot);
        total += partitionSnapshot.getValue();
    }
    StatsSnapshot nodeSnapshot = new StatsSnapshot(total, partitionMap);
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.QUOTA, timestamp, numbOfPartitions, numbOfPartitions, Collections.EMPTY_LIST);
    return new StatsWrapper(header, nodeSnapshot);
}

Example 2 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class TestUtils method generateNodeStats.

/**
 * Given a {@link List} of {@link StatsSnapshot}s and a timestamp generate a {@link StatsWrapper} that would have been
 * produced by a node.
 * @param storeSnapshots a {@link List} of store level {@link StatsSnapshot}s.
 * @param timestamp the timestamp to be attached to the generated {@link StatsWrapper}
 * @param type the type of stats report to generate on this node
 * @return the generated node level {@link StatsWrapper}
 */
public static StatsWrapper generateNodeStats(List<StatsSnapshot> storeSnapshots, long timestamp, StatsReportType type) {
    long total = 0;
    int numbOfPartitions = storeSnapshots.size();
    Map<String, StatsSnapshot> partitionMap = new HashMap<>();
    Map<String, StatsSnapshot> partitionClassMap = new HashMap<>();
    String[] PARTITION_CLASS = new String[] { "PartitionClass1", "PartitionClass2" };
    for (int i = 0; i < numbOfPartitions; i++) {
        String partitionIdStr = Utils.statsPartitionKey(i);
        StatsSnapshot partitionSnapshot = storeSnapshots.get(i);
        partitionMap.put(partitionIdStr, partitionSnapshot);
        total += partitionSnapshot.getValue();
        if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            String partitionClassStr = PARTITION_CLASS[i % PARTITION_CLASS.length];
            StatsSnapshot partitionClassSnapshot = partitionClassMap.getOrDefault(partitionClassStr, new StatsSnapshot(0L, new HashMap<>()));
            partitionClassSnapshot.setValue(partitionClassSnapshot.getValue() + partitionSnapshot.getValue());
            partitionClassSnapshot.getSubMap().put(partitionIdStr, partitionSnapshot);
            partitionClassMap.put(partitionClassStr, partitionClassSnapshot);
        }
    }
    StatsSnapshot nodeSnapshot = null;
    if (type == StatsReportType.ACCOUNT_REPORT) {
        nodeSnapshot = new StatsSnapshot(total, partitionMap);
    } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
        nodeSnapshot = new StatsSnapshot(total, partitionClassMap);
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, timestamp, numbOfPartitions, numbOfPartitions, Collections.emptyList());
    return new StatsWrapper(header, nodeSnapshot);
}

Example 3 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithDiffNodeStats.

/**
 * Tests to verify cluster aggregation with node stats that contain different partition stats.
 * @throws IOException
 */
@Test
public void testDoWorkWithDiffNodeStats() throws IOException {
    long seed = 1234;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> greaterStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> smallerStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> mediumStoreSnapshots = new ArrayList<>();
        greaterStoreSnapshots.add(TestUtils.generateStoreStats(6, 3, new Random(seed), type));
        mediumStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        smallerStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        StatsWrapper greaterNodeStats = TestUtils.generateNodeStats(greaterStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper mediumNodeStats = TestUtils.generateNodeStats(mediumStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper smallerNodeStats = TestUtils.generateNodeStats(smallerStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        Map<String, String> instanceToStatsMap = new LinkedHashMap<>();
        instanceToStatsMap.put("Instance_0", mapper.writeValueAsString(smallerNodeStats));
        instanceToStatsMap.put("Instance_1", mapper.writeValueAsString(greaterNodeStats));
        instanceToStatsMap.put("Instance_2", mapper.writeValueAsString(mediumNodeStats));
        instanceToStatsMap.put("Instance_3", mapper.writeValueAsString(emptyNodeStats));
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, new HashMap<>());
        StatsSnapshot expectedValidSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterStoreSnapshots.get(0));
                expectedValidSnapshot = greaterStoreSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                expectedValidSnapshot = HelixClusterAggregator.reduceByPartitionClass(greaterNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterNodeStats.getSnapshot());
                expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
                break;
        }
        // verify cluster wide aggregation on raw data with different node stats
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertTrue("Mismatch in the raw aggregated snapshot for " + type, expectedRawSnapshot.equals(rawSnapshot));
        // verify cluster wide aggregation on valid data with different node stats
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the valid aggregated snapshot for " + type, expectedValidSnapshot.equals(actualSnapshot));
        // verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 4 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkBasic.

/**
 * Basic tests to verify the cluster wide raw data and valid data aggregation. The tests also verify stats aggregation
 * for all types of stats reports.
 * @throws IOException
 */
@Test
public void testDoWorkBasic() throws IOException {
    int nodeCount = 3;
    Random random = new Random();
    // For each type of report, create snapshots for 3 stores with 3 accounts, 4 accounts and 5 accounts respectively.
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> storeSnapshots = new ArrayList<>();
        for (int i = 3; i < 6; i++) {
            storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
        }
        StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, DEFAULT_TIMESTAMP, type);
        String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        // selects the replica with highest value.
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, nodeStatsJSON);
        }
        // Add two special cases into instance-to-stats map for testing:
        // (1) empty stats report from certain instance
        // (2) corrupted/invalid stats report from certain instance (this is simulated by empty string)
        instanceToStatsMap.put("Instance_" + nodeCount, emptyStatsJSON);
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        // 1. Aggregate all snapshots into the first snapshot in snapshots list. The intention is to get expected aggregated snapshot.
        // 2. Then invoke clusterAggregator to do work on stats across all instances.
        // 3. Verify both raw stats and valid stats after aggregation
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                // we expect for valid data aggregation.
                for (int i = 1; i < storeSnapshots.size(); i++) {
                    StatsSnapshot.aggregate(storeSnapshots.get(0), storeSnapshots.get(i));
                }
                expectedSnapshot = storeSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                // Invoke reduceByPartitionClass to remove partition level and only keep the partition class and account_container entries
                expectedSnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStats.getSnapshot());
                break;
        }
        // Verify cluster wide raw stats aggregation
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertEquals("Mismatch in total value of " + type, nodeCount * expectedSnapshot.getValue(), rawSnapshot.getValue());
        if (type == StatsReportType.ACCOUNT_REPORT) {
            verifyAggregatedRawStatsForAccountReport(rawSnapshot, expectedSnapshot, nodeCount);
        } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            verifyAggregatedRawStatsForPartitionClassReport(rawSnapshot, expectedSnapshot, nodeCount);
        }
        // Verify cluster wide stats aggregation
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated snapshot", expectedSnapshot.equals(actualSnapshot));
        // Verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 5 with StatsWrapper

use of com.github.ambry.server.StatsWrapper in project ambry by linkedin.

the class HelixClusterAggregatorTest method testStatsAggregationWithAllEmptyNodes.

/**
 * Tests to verify cluster aggregation with all empty nodes.
 * @throws IOException
 */
@Test
public void testStatsAggregationWithAllEmptyNodes() throws IOException {
    int nodeCount = 3;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, emptyStatsJSON);
        }
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = new StatsSnapshot(0L, null);
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        StatsSnapshot validSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in raw snapshot", expectedSnapshot.equals(rawSnapshot));
        assertTrue("Mismatch in valid snapshot", expectedSnapshot.equals(validSnapshot));
    }
}