Examples with StatsSnapshot com.github.ambry.server.StatsSnapshot used on opensource projects

Example 6 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkWithDiffNodeStats.

/**
 * Tests to verify cluster aggregation with node stats that contain different partition stats.
 * @throws IOException
 */
@Test
public void testDoWorkWithDiffNodeStats() throws IOException {
    long seed = 1234;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> greaterStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> smallerStoreSnapshots = new ArrayList<>();
        List<StatsSnapshot> mediumStoreSnapshots = new ArrayList<>();
        greaterStoreSnapshots.add(TestUtils.generateStoreStats(6, 3, new Random(seed), type));
        mediumStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        smallerStoreSnapshots.add(TestUtils.generateStoreStats(5, 3, new Random(seed), type));
        StatsWrapper greaterNodeStats = TestUtils.generateNodeStats(greaterStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper mediumNodeStats = TestUtils.generateNodeStats(mediumStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper smallerNodeStats = TestUtils.generateNodeStats(smallerStoreSnapshots, DEFAULT_TIMESTAMP, type);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        Map<String, String> instanceToStatsMap = new LinkedHashMap<>();
        instanceToStatsMap.put("Instance_0", mapper.writeValueAsString(smallerNodeStats));
        instanceToStatsMap.put("Instance_1", mapper.writeValueAsString(greaterNodeStats));
        instanceToStatsMap.put("Instance_2", mapper.writeValueAsString(mediumNodeStats));
        instanceToStatsMap.put("Instance_3", mapper.writeValueAsString(emptyNodeStats));
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedRawSnapshot = new StatsSnapshot(0L, new HashMap<>());
        StatsSnapshot expectedValidSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumStoreSnapshots.get(0));
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterStoreSnapshots.get(0));
                expectedValidSnapshot = greaterStoreSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                expectedValidSnapshot = HelixClusterAggregator.reduceByPartitionClass(greaterNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, mediumNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, smallerNodeStats.getSnapshot());
                StatsSnapshot.aggregate(expectedRawSnapshot, greaterNodeStats.getSnapshot());
                expectedRawSnapshot = HelixClusterAggregator.reduceByPartitionClass(expectedRawSnapshot);
                break;
        }
        // verify cluster wide aggregation on raw data with different node stats
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertTrue("Mismatch in the raw aggregated snapshot for " + type, expectedRawSnapshot.equals(rawSnapshot));
        // verify cluster wide aggregation on valid data with different node stats
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the valid aggregated snapshot for " + type, expectedValidSnapshot.equals(actualSnapshot));
        // verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 7 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregatorTest method testDoWorkBasic.

/**
 * Basic tests to verify the cluster wide raw data and valid data aggregation. The tests also verify stats aggregation
 * for all types of stats reports.
 * @throws IOException
 */
@Test
public void testDoWorkBasic() throws IOException {
    int nodeCount = 3;
    Random random = new Random();
    // For each type of report, create snapshots for 3 stores with 3 accounts, 4 accounts and 5 accounts respectively.
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        List<StatsSnapshot> storeSnapshots = new ArrayList<>();
        for (int i = 3; i < 6; i++) {
            storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
        }
        StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, DEFAULT_TIMESTAMP, type);
        String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        // selects the replica with highest value.
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, nodeStatsJSON);
        }
        // Add two special cases into instance-to-stats map for testing:
        // (1) empty stats report from certain instance
        // (2) corrupted/invalid stats report from certain instance (this is simulated by empty string)
        instanceToStatsMap.put("Instance_" + nodeCount, emptyStatsJSON);
        instanceToStatsMap.put(EXCEPTION_INSTANCE_NAME, "");
        // 1. Aggregate all snapshots into the first snapshot in snapshots list. The intention is to get expected aggregated snapshot.
        // 2. Then invoke clusterAggregator to do work on stats across all instances.
        // 3. Verify both raw stats and valid stats after aggregation
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = null;
        switch(type) {
            case ACCOUNT_REPORT:
                // we expect for valid data aggregation.
                for (int i = 1; i < storeSnapshots.size(); i++) {
                    StatsSnapshot.aggregate(storeSnapshots.get(0), storeSnapshots.get(i));
                }
                expectedSnapshot = storeSnapshots.get(0);
                break;
            case PARTITION_CLASS_REPORT:
                // Invoke reduceByPartitionClass to remove partition level and only keep the partition class and account_container entries
                expectedSnapshot = HelixClusterAggregator.reduceByPartitionClass(nodeStats.getSnapshot());
                break;
        }
        // Verify cluster wide raw stats aggregation
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        assertEquals("Mismatch in total value of " + type, nodeCount * expectedSnapshot.getValue(), rawSnapshot.getValue());
        if (type == StatsReportType.ACCOUNT_REPORT) {
            verifyAggregatedRawStatsForAccountReport(rawSnapshot, expectedSnapshot, nodeCount);
        } else if (type == StatsReportType.PARTITION_CLASS_REPORT) {
            verifyAggregatedRawStatsForPartitionClassReport(rawSnapshot, expectedSnapshot, nodeCount);
        }
        // Verify cluster wide stats aggregation
        StatsSnapshot actualSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in the aggregated snapshot", expectedSnapshot.equals(actualSnapshot));
        // Verify aggregator keeps track of instances where exception occurred.
        assertEquals("Mismatch in instances where exception occurred", Collections.singletonList(EXCEPTION_INSTANCE_NAME), clusterAggregator.getExceptionOccurredInstances(type));
    }
}

Example 8 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixClusterAggregatorTest method testStatsAggregationWithAllEmptyNodes.

/**
 * Tests to verify cluster aggregation with all empty nodes.
 * @throws IOException
 */
@Test
public void testStatsAggregationWithAllEmptyNodes() throws IOException {
    int nodeCount = 3;
    for (StatsReportType type : EnumSet.of(StatsReportType.ACCOUNT_REPORT, StatsReportType.PARTITION_CLASS_REPORT)) {
        StatsWrapper emptyNodeStats = TestUtils.generateNodeStats(Collections.emptyList(), DEFAULT_TIMESTAMP, type);
        String emptyStatsJSON = mapper.writeValueAsString(emptyNodeStats);
        Map<String, String> instanceToStatsMap = new HashMap<>();
        for (int i = 0; i < nodeCount; i++) {
            instanceToStatsMap.put("Instance_" + i, emptyStatsJSON);
        }
        Pair<StatsSnapshot, StatsSnapshot> aggregatedRawAndValidStats = clusterAggregator.doWork(instanceToStatsMap, type);
        StatsSnapshot expectedSnapshot = new StatsSnapshot(0L, null);
        StatsSnapshot rawSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getFirst()), StatsSnapshot.class);
        StatsSnapshot validSnapshot = mapper.readValue(mapper.writeValueAsString(aggregatedRawAndValidStats.getSecond()), StatsSnapshot.class);
        assertTrue("Mismatch in raw snapshot", expectedSnapshot.equals(rawSnapshot));
        assertTrue("Mismatch in valid snapshot", expectedSnapshot.equals(validSnapshot));
    }
}

Example 9 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class HelixHealthReportAggregationTaskTest method initializeNodeReports.

/**
 * Initialize the reports and create instances in helix if not exists.
 * @param type The type of reports to create
 * @param numNode The number of nodes to initiate.
 * @param startingPort The starting port number, which will then be incremented to represent different nodes.
 * @throws IOException
 */
private void initializeNodeReports(StatsReportType type, int numNode, int startingPort) throws IOException {
    String healthReportName = type == StatsReportType.ACCOUNT_REPORT ? HEALTH_REPORT_NAME_ACCOUNT : HEALTH_REPORT_NAME_PARTITION;
    String statsFieldName = type == StatsReportType.ACCOUNT_REPORT ? STATS_FIELD_NAME_ACCOUNT : STATS_FIELD_NAME_PARTITION;
    List<StatsSnapshot> storeSnapshots = new ArrayList<>();
    Random random = new Random();
    for (int i = 3; i < 6; i++) {
        storeSnapshots.add(TestUtils.generateStoreStats(i, 3, random, type));
    }
    StatsWrapper nodeStats = TestUtils.generateNodeStats(storeSnapshots, 1000, type);
    String nodeStatsJSON = mapper.writeValueAsString(nodeStats);
    HelixDataAccessor dataAccessor = mockHelixManager.getHelixDataAccessor();
    for (int i = 0; i < numNode; i++) {
        String instanceName = ClusterMapUtils.getInstanceName("localhost", startingPort);
        InstanceConfig instanceConfig = new InstanceConfig(instanceName);
        instanceConfig.setHostName("localhost");
        instanceConfig.setPort(Integer.toString(startingPort));
        mockHelixAdmin.addInstance(CLUSTER_NAME, instanceConfig);
        PropertyKey key = dataAccessor.keyBuilder().healthReport(instanceName, healthReportName);
        ZNRecord znRecord = new ZNRecord(instanceName);
        // Set the same reports for all instances
        znRecord.setSimpleField(statsFieldName, nodeStatsJSON);
        HelixProperty helixProperty = new HelixProperty(znRecord);
        dataAccessor.setProperty(key, helixProperty);
        startingPort++;
    }
}

Example 10 with StatsSnapshot

use of com.github.ambry.server.StatsSnapshot in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testHostPartitionClassStorageStats.

/**
 * Test methods to store and fetch partition class, partition name partition id and partition class storage stats.
 * @throws Exception
 */
@Test
public void testHostPartitionClassStorageStats() throws Exception {
    // First write some stats to account reports
    testMultiStoreStats();
    HostAccountStorageStatsWrapper accountStats1 = mySqlStore.queryHostAccountStorageStatsByHost(hostname1, port1);
    HostAccountStorageStatsWrapper accountStats2 = mySqlStore.queryHostAccountStorageStatsByHost(hostname2, port2);
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    HostAccountStorageStatsWrapper accountStats3 = mySqlStore3.queryHostAccountStorageStatsByHost(hostname3, port3);
    // From this account stats, create partition class stats;
    Set<Long> allPartitionKeys = new HashSet<Long>() {

        {
            addAll(accountStats1.getStats().getStorageStats().keySet());
            addAll(accountStats2.getStats().getStorageStats().keySet());
            addAll(accountStats3.getStats().getStorageStats().keySet());
        }
    };
    List<String> partitionClassNames = Arrays.asList("default", "new");
    Map<Long, String> partitionIdToClassName = new HashMap<>();
    int ind = 0;
    for (long partitionId : allPartitionKeys) {
        partitionIdToClassName.put(partitionId, partitionClassNames.get(ind % partitionClassNames.size()));
        ind++;
    }
    HostPartitionClassStorageStatsWrapper partitionClassStats1 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats1, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats2 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats2, partitionIdToClassName);
    HostPartitionClassStorageStatsWrapper partitionClassStats3 = convertHostAccountStorageStatsToHostPartitionClassStorageStats(accountStats3, partitionIdToClassName);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats1);
    mySqlStore.storeHostPartitionClassStorageStats(partitionClassStats2);
    mySqlStore3.storeHostPartitionClassStorageStats(partitionClassStats3);
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    assertEquals(new HashSet<>(partitionClassNames), partitionNameAndIds.keySet());
    Map<Long, String> dbPartitionKeyToClassName = partitionNameAndIds.entrySet().stream().flatMap(ent -> ent.getValue().stream().map(pid -> new Pair<>(ent.getKey(), (long) pid))).collect(Collectors.toMap(Pair::getSecond, Pair::getFirst));
    assertEquals(partitionIdToClassName, dbPartitionKeyToClassName);
    // Fetch HostPartitionClassStorageStats
    HostPartitionClassStorageStatsWrapper obtainedStats1 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(partitionClassStats1.getStats().getStorageStats(), obtainedStats1.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats2 = mySqlStore.queryHostPartitionClassStorageStatsByHost(hostname2, port2, partitionNameAndIds);
    assertEquals(partitionClassStats2.getStats().getStorageStats(), obtainedStats2.getStats().getStorageStats());
    HostPartitionClassStorageStatsWrapper obtainedStats3 = mySqlStore3.queryHostPartitionClassStorageStatsByHost(hostname3, port3, partitionNameAndIds);
    assertEquals(partitionClassStats3.getStats().getStorageStats(), obtainedStats3.getStats().getStorageStats());
    // Fetch StatsSnapshot
    StatsWrapper obtainedStats = mySqlStore.queryPartitionClassStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(StorageStatsUtil.convertHostPartitionClassStorageStatsToStatsSnapshot(obtainedStats1.getStats(), false), obtainedStats.getSnapshot());
    mySqlStore3.shutdown();
}