Examples with Symptom com.microsoft.dhalion.core.Symptom used on opensource projects

Example 1 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class GrowingWaitQueueDetector method detect.

/**
 * Detects all components unable to keep up with input load, hence having a growing pending buffer
 * or wait queue
 *
 * @return A collection of symptoms each one corresponding to a components executing slower
 * than input rate.
 */
@Override
public Collection<Symptom> detect(Collection<Measurement> measurements) {
    Collection<Symptom> result = new ArrayList<>();
    MeasurementsTable waitQueueMetrics = MeasurementsTable.of(measurements).type(METRIC_WAIT_Q_SIZE.text());
    for (String component : waitQueueMetrics.uniqueComponents()) {
        double maxSlope = computeWaitQueueSizeTrend(waitQueueMetrics.component(component));
        if (maxSlope > rateLimit) {
            LOG.info(String.format("Detected growing wait queues for %s, max rate %f", component, maxSlope));
            Collection<String> addresses = Collections.singletonList(component);
            result.add(new Symptom(SYMPTOM_GROWING_WAIT_Q.text(), context.checkpoint(), addresses));
        }
    }
    return result;
}

Example 2 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class LargeWaitQueueDetector method detect.

/**
 * Detects all components having a large pending buffer or wait queue
 *
 * @return A collection of symptoms each one corresponding to components with
 * large wait queues.
 */
@Override
public Collection<Symptom> detect(Collection<Measurement> measurements) {
    Collection<Symptom> result = new ArrayList<>();
    MeasurementsTable waitQueueMetrics = MeasurementsTable.of(measurements).type(METRIC_WAIT_Q_SIZE.text());
    for (String component : waitQueueMetrics.uniqueComponents()) {
        Set<String> addresses = new HashSet<>();
        MeasurementsTable instanceMetrics = waitQueueMetrics.component(component);
        for (String instance : instanceMetrics.uniqueInstances()) {
            double avgWaitQSize = instanceMetrics.instance(instance).mean();
            if (avgWaitQSize > sizeLimit) {
                LOG.info(String.format("Detected large wait queues for instance" + "%s, smallest queue is + %f", instance, avgWaitQSize));
                addresses.add(instance);
            }
        }
        if (addresses.size() > 0) {
            result.add(new Symptom(SYMPTOM_LARGE_WAIT_Q.text(), context.checkpoint(), addresses));
        }
    }
    return result;
}

Example 3 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class GrowingWaitQueueDetectorTest method testDetector.

@Test
public void testDetector() {
    HealthPolicyConfig config = mock(HealthPolicyConfig.class);
    when(config.getConfig(CONF_LIMIT, 10.0)).thenReturn(5.0);
    Measurement measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 0.0);
    Measurement measurement2 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892270), 300.0);
    Measurement measurement3 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892330), 700.0);
    Measurement measurement4 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892390), 1000.0);
    Measurement measurement5 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892450), 1300.0);
    Collection<Measurement> metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    metrics.add(measurement3);
    metrics.add(measurement4);
    metrics.add(measurement5);
    GrowingWaitQueueDetector detector = new GrowingWaitQueueDetector(config);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(Instant.now());
    detector.initialize(context);
    Collection<Symptom> symptoms = detector.detect(metrics);
    assertEquals(1, symptoms.size());
    assertEquals(1, symptoms.iterator().next().assignments().size());
    measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 0.0);
    measurement2 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892270), 200.0);
    measurement3 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892330), 400.0);
    measurement4 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892390), 600.0);
    measurement5 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892450), 800.0);
    metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    metrics.add(measurement3);
    metrics.add(measurement4);
    metrics.add(measurement5);
    detector = new GrowingWaitQueueDetector(config);
    symptoms = detector.detect(metrics);
    assertEquals(0, symptoms.size());
}

Example 4 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class BackPressureDetectorTest method testConfigAndFilter.

@Test
public void testConfigAndFilter() throws IOException {
    HealthPolicyConfig config = mock(HealthPolicyConfig.class);
    when(config.getConfig(CONF_NOISE_FILTER, 20)).thenReturn(50);
    Measurement measurement1 = new Measurement("bolt", "i1", METRIC_BACK_PRESSURE.text(), now, 55);
    Measurement measurement2 = new Measurement("bolt", "i2", METRIC_BACK_PRESSURE.text(), now, 3);
    Measurement measurement3 = new Measurement("bolt", "i3", METRIC_BACK_PRESSURE.text(), now, 0);
    Collection<Measurement> metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    metrics.add(measurement3);
    HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
    BackPressureDetector detector = new BackPressureDetector(config, publishingMetrics);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(now);
    detector.initialize(context);
    Collection<Symptom> symptoms = detector.detect(metrics);
    Assert.assertEquals(2, symptoms.size());
    SymptomsTable compSymptom = SymptomsTable.of(symptoms).type(SYMPTOM_COMP_BACK_PRESSURE.text());
    Assert.assertEquals(1, compSymptom.size());
    Assert.assertEquals(1, compSymptom.get().iterator().next().assignments().size());
    SymptomsTable instanceSymptom = SymptomsTable.of(symptoms).type(SYMPTOM_INSTANCE_BACK_PRESSURE.text());
    Assert.assertEquals(1, instanceSymptom.size());
    Assert.assertEquals(1, instanceSymptom.get().iterator().next().assignments().size());
    Symptom symptom = symptoms.iterator().next();
    measurement1 = new Measurement("bolt", "i1", METRIC_BACK_PRESSURE.text(), now, 45);
    measurement2 = new Measurement("bolt", "i2", METRIC_BACK_PRESSURE.text(), now, 3);
    metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    detector = new BackPressureDetector(config, publishingMetrics);
    detector.initialize(context);
    symptoms = detector.detect(metrics);
    Assert.assertEquals(0, symptoms.size());
}

Example 5 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class LargeWaitQueueDetectorTest method testConfigAndFilter.

@Test
public void testConfigAndFilter() {
    HealthPolicyConfig config = mock(HealthPolicyConfig.class);
    when(config.getConfig(CONF_SIZE_LIMIT, 1000)).thenReturn(20);
    Measurement measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 21);
    Measurement measurement2 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892322), 21);
    Collection<Measurement> metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    LargeWaitQueueDetector detector = new LargeWaitQueueDetector(config);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(Instant.now());
    detector.initialize(context);
    Collection<Symptom> symptoms = detector.detect(metrics);
    assertEquals(1, symptoms.size());
    assertEquals(1, symptoms.iterator().next().assignments().size());
    measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 11);
    measurement2 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892322), 10);
    metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    detector = new LargeWaitQueueDetector(config);
    symptoms = detector.detect(metrics);
    assertEquals(0, symptoms.size());
}