Examples with StatsReportType com.github.ambry.server.StatsReportType used on opensource projects

Example 1 with StatsReportType

use of com.github.ambry.server.StatsReportType in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithDiffNodeStats.

/**
 * Tests to verify cluster aggregation with node stats that contain different partition stats.
 * @throws IOException
 */
@Test
public void testDoWorkWithDiffNodeStats() throws IOException {
    long seed = 1234;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> greaterStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> smallerStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> mediumStoreSnapshots = new ArrayList<>();
        greaterStoreSnapshots.add(TestUtils.generateStoreStats(6, 3, new Random(seed), type));
        mediumStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        smallerStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        StatsWrapper greaterNodeStats = TestUtils.generateNodeStats(greaterStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper mediumNodeStats = TestUtils.generateNodeStats(mediumStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper smallerNodeStats = TestUtils.generateNodeStats(smallerStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        Map<String, String> instanceToStatsMap = new LinkedHashMap<>();
        instanceToStatsMap.put("Instance_0", mapper.writeValueAsString(smallerNodeStats));
        instanceToStatsMap.put("Instance_1", mapper.writeValueAsString(greaterNodeStats));
        instanceToStatsMap.put("Instance_2", mapper.writeValueAsString(mediumNodeStats));
        instanceToStatsMap.put("Instance_3", mapper.writeValueAsString(emptyNodeStats));
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, new HashMap<>());
        StatsSnapshot expectedValidSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterStoreSnapshots.get(0));
                expectedValidSnapshot = greaterStoreSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                expectedValidSnapshot = HelixClusterAggregator.reduceByPartitionClass(greaterNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterNodeStats.getSnapshot());
                expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
                break;
        }
        // verify cluster wide aggregation on raw data with different node stats
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertTrue("Mismatch in the raw aggregated snapshot for " + type, expectedRawSnapshot.equals(rawSnapshot));
        // verify cluster wide aggregation on valid data with different node stats
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the valid aggregated snapshot for " + type, expectedValidSnapshot.equals(actualSnapshot));
        // verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 2 with StatsReportType

use of com.github.ambry.server.StatsReportType in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkBasic.

/**
 * Basic tests to verify the cluster wide raw data and valid data aggregation. The tests also verify stats aggregation
 * for all types of stats reports.
 * @throws IOException
 */
@Test
public void testDoWorkBasic() throws IOException {
    int nodeCount = 3;
    Random random = new Random();
    // For each type of report, create snapshots for 3 stores with 3 accounts, 4 accounts and 5 accounts respectively.
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> storeSnapshots = new ArrayList<>();
        for (int i = 3; i < 6; i++) {
            storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
        }
        StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, DEFAULT_TIMESTAMP, type);
        String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        // selects the replica with highest value.
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, nodeStatsJSON);
        }
        // Add two special cases into instance-to-stats map for testing:
        // (1) empty stats report from certain instance
        // (2) corrupted/invalid stats report from certain instance (this is simulated by empty string)
        instanceToStatsMap.put("Instance_" + nodeCount, emptyStatsJSON);
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        // 1. Aggregate all snapshots into the first snapshot in snapshots list. The intention is to get expected aggregated snapshot.
        // 2. Then invoke clusterAggregator to do work on stats across all instances.
        // 3. Verify both raw stats and valid stats after aggregation
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                // we expect for valid data aggregation.
                for (int i = 1; i < storeSnapshots.size(); i++) {
                    StatsSnapshot.aggregate(storeSnapshots.get(0), storeSnapshots.get(i));
                }
                expectedSnapshot = storeSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                // Invoke reduceByPartitionClass to remove partition level and only keep the partition class and account_container entries
                expectedSnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStats.getSnapshot());
                break;
        }
        // Verify cluster wide raw stats aggregation
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertEquals("Mismatch in total value of " + type, nodeCount * expectedSnapshot.getValue(), rawSnapshot.getValue());
        if (type == StatsReportType.ACCOUNT_REPORT) {
            verifyAggregatedRawStatsForAccountReport(rawSnapshot, expectedSnapshot, nodeCount);
        } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            verifyAggregatedRawStatsForPartitionClassReport(rawSnapshot, expectedSnapshot, nodeCount);
        }
        // Verify cluster wide stats aggregation
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated snapshot", expectedSnapshot.equals(actualSnapshot));
        // Verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 3 with StatsReportType

use of com.github.ambry.server.StatsReportType in project ambry by linkedin.

the class HelixClusterAggregatorTest method testStatsAggregationWithAllEmptyNodes.

/**
 * Tests to verify cluster aggregation with all empty nodes.
 * @throws IOException
 */
@Test
public void testStatsAggregationWithAllEmptyNodes() throws IOException {
    int nodeCount = 3;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, emptyStatsJSON);
        }
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = new StatsSnapshot(0L, null);
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        StatsSnapshot validSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in raw snapshot", expectedSnapshot.equals(rawSnapshot));
        assertTrue("Mismatch in valid snapshot", expectedSnapshot.equals(validSnapshot));
    }
}

Example 4 with StatsReportType

use of com.github.ambry.server.StatsReportType in project ambry by linkedin.

the class HelixClusterAggregatorTest method testStatsAggregationWithZeroValueSnapshots.

/**
 * Test removing zero value snapshots from aggregated result.
 * @throws IOException
 */
@Test
public void testStatsAggregationWithZeroValueSnapshots() throws IOException {
    int nodeCount = 3;
    Random random = new Random();
    // For each type of report, create snapshots for 3 stores with 3 accounts, 4 accounts and 5 accounts respectively.
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> storeSnapshots = new ArrayList<>();
        for (int i = 3; i < 6; i++) {
            storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
        }
        // add zero value to the first snapshot
        if (type == StatsReportType.ACCOUNT_REPORT) {
            Map<String, StatsSnapshot> accountStatsSnapshotMap = storeSnapshots.get(0).getSubMap();
            accountStatsSnapshotMap.put("A[100]", new StatsSnapshot(0L, new HashMap<String, StatsSnapshot>() {

                {
                    put("C[100]", new StatsSnapshot(0L, null));
                }
            }));
        } else {
            Map<String, StatsSnapshot> accountContainerStatsSnapshotMap = storeSnapshots.get(0).getSubMap();
            accountContainerStatsSnapshotMap.put("A[100]_C[100]", new StatsSnapshot(0L, null));
        }
        StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, DEFAULT_TIMESTAMP, type);
        String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, nodeStatsJSON);
        }
        // 1. Aggregate all snapshots into the first snapshot in snapshots list. The intention is to get expected aggregated snapshot.
        // 2. Then invoke clusterAggregator to do work on stats across all instances.
        // 3. Verify both raw stats and valid stats after aggregation
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        // Remove 0 values snapshots first
        StatsSnapshot expectedSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                storeSnapshots.get(0).getSubMap().remove("A[100]");
                // we expect for valid data aggregation.
                for (int i = 1; i < storeSnapshots.size(); i++) {
                    StatsSnapshot.aggregate(storeSnapshots.get(0), storeSnapshots.get(i));
                }
                expectedSnapshot = storeSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                storeSnapshots.get(0).getSubMap().remove("A[100]_C[100]");
                // Invoke reduceByPartitionClass to remove partition level and only keep the partition class and account_container entries
                expectedSnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStats.getSnapshot());
                break;
        }
        // Verify cluster wide raw stats aggregation
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertEquals("Mismatch in total value of " + type, nodeCount * expectedSnapshot.getValue(), rawSnapshot.getValue());
        if (type == StatsReportType.ACCOUNT_REPORT) {
            verifyAggregatedRawStatsForAccountReport(rawSnapshot, expectedSnapshot, nodeCount);
        } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            verifyAggregatedRawStatsForPartitionClassReport(rawSnapshot, expectedSnapshot, nodeCount);
        }
        // Verify cluster wide stats aggregation
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated snapshot", expectedSnapshot.equals(actualSnapshot));
    }
}

Example 5 with StatsReportType

use of com.github.ambry.server.StatsReportType in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithOutdatedNode.

/**
 * Tests to verify cluster wide aggregation with outdated node stats.
 * @throws IOException
 */
@Test
public void testDoWorkWithOutdatedNode() throws IOException {
    long seed = 1111;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> upToDateStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> outdatedStoreSnapshots = new ArrayList<>();
        upToDateStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        outdatedStoreSnapshots.add(TestUtils.generateStoreStats(6, 3, new Random(seed), type));
        StatsWrapper upToDateNodeStats = TestUtils.generateNodeStats(upToDateStoreSnapshots, TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), type);
        StatsWrapper outdatedNodeStats = TestUtils.generateNodeStats(outdatedStoreSnapshots, 0, type);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), TimeUnit.MINUTES.toMillis(2 * RELEVANT_PERIOD_IN_MINUTES), type);
        Map<String, String> instanceToStatsMap = new LinkedHashMap<>();
        instanceToStatsMap.put("Instance_0", mapper.writeValueAsString(outdatedNodeStats));
        instanceToStatsMap.put("Instance_1", mapper.writeValueAsString(upToDateNodeStats));
        instanceToStatsMap.put("Instance_2", mapper.writeValueAsString(emptyNodeStats));
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedValidSnapshot = null;
        StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, new HashMap<>());
        switch(type) {
            case ACCOUNT_REPORT:
                expectedValidSnapshot = upToDateStoreSnapshots.get(0);
                StatsSnapshot.aggregate(expectedRawSnapshot, outdatedStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, upToDateStoreSnapshots.get(0));
                break;
            case PARTITION_CLASS_REPORT:
                expectedValidSnapshot = HelixClusterAggregator.reduceByPartitionClass(upToDateNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, outdatedNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, upToDateNodeStats.getSnapshot());
                expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
                break;
        }
        // verify cluster wide aggregation on raw stats with outdated node stats
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated raw snapshot", expectedRawSnapshot.equals(rawSnapshot));
        // verify cluster wide aggregation on valid stats with outdated node stats
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated valid snapshot", expectedValidSnapshot.equals(actualSnapshot));
        // verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}