Examples with Symptom com.microsoft.dhalion.core.Symptom used on opensource projects

Example 16 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class SlowInstanceDiagnoserTest method failIfInstanceWithBpHasSmallBuffer.

@Test
public void failIfInstanceWithBpHasSmallBuffer() {
    Collection<String> assign = Collections.singleton(comp);
    Symptom bpSymptom = new Symptom(SYMPTOM_COMP_BACK_PRESSURE.text(), now, assign);
    Symptom qDisparitySymptom = new Symptom(SYMPTOM_WAIT_Q_SIZE_SKEW.text(), now, assign);
    Symptom exeDisparitySymptom = new Symptom(SYMPTOM_PROCESSING_RATE_SKEW.text(), now, assign);
    Collection<Symptom> symptoms = Arrays.asList(bpSymptom, qDisparitySymptom, exeDisparitySymptom);
    Collection<Diagnosis> result = diagnoser.diagnose(symptoms);
    assertEquals(0, result.size());
}

Example 17 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class WaitQueueSkewDetectorTest method testConfigAndFilter.

@Test
public void testConfigAndFilter() {
    HealthPolicyConfig config = mock(HealthPolicyConfig.class);
    when(config.getConfig(CONF_SKEW_RATIO, 20.0)).thenReturn(15.0);
    Measurement measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 1501);
    Measurement measurement2 = new Measurement("bolt", "i2", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 100.0);
    Collection<Measurement> metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    WaitQueueSkewDetector detector = new WaitQueueSkewDetector(config);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(Instant.now());
    detector.initialize(context);
    Collection<Symptom> symptoms = detector.detect(metrics);
    assertEquals(3, symptoms.size());
    SymptomsTable symptomsTable = SymptomsTable.of(symptoms);
    assertEquals(1, symptomsTable.type("POSITIVE " + BaseDetector.SymptomType.SYMPTOM_WAIT_Q_SIZE_SKEW).size());
    assertEquals(1, symptomsTable.type("NEGATIVE " + BaseDetector.SymptomType.SYMPTOM_WAIT_Q_SIZE_SKEW).size());
    assertEquals(1, symptomsTable.type("POSITIVE " + BaseDetector.SymptomType.SYMPTOM_WAIT_Q_SIZE_SKEW).assignment("i1").size());
    assertEquals(1, symptomsTable.type("NEGATIVE " + BaseDetector.SymptomType.SYMPTOM_WAIT_Q_SIZE_SKEW).assignment("i2").size());
    measurement1 = new Measurement("bolt", "i1", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 1500);
    measurement2 = new Measurement("bolt", "i2", METRIC_WAIT_Q_SIZE.text(), Instant.ofEpochSecond(1497892222), 110.0);
    metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    detector = new WaitQueueSkewDetector(config);
    detector.initialize(context);
    symptoms = detector.detect(metrics);
    assertEquals(0, symptoms.size());
}

Example 18 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class ProcessingRateSkewDetectorTest method testReturnsMultipleComponents.

@Test
public void testReturnsMultipleComponents() {
    HealthPolicyConfig config = mock(HealthPolicyConfig.class);
    when(config.getConfig(CONF_SKEW_RATIO, 1.5)).thenReturn(2.5);
    Measurement measurement1 = new Measurement("bolt", "i1", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 1000);
    Measurement measurement2 = new Measurement("bolt", "i2", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 200.0);
    Measurement measurement3 = new Measurement("bolt2", "i3", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 1000);
    Measurement measurement4 = new Measurement("bolt2", "i4", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 200.0);
    Measurement measurement5 = new Measurement("bolt3", "i5", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 1000);
    Measurement measurement6 = new Measurement("bolt3", "i6", METRIC_EXE_COUNT.text(), Instant.ofEpochSecond(1497892222), 500.0);
    Collection<Measurement> metrics = new ArrayList<>();
    metrics.add(measurement1);
    metrics.add(measurement2);
    metrics.add(measurement3);
    metrics.add(measurement4);
    metrics.add(measurement5);
    metrics.add(measurement6);
    ProcessingRateSkewDetector detector = new ProcessingRateSkewDetector(config);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(Instant.now());
    detector.initialize(context);
    Collection<Symptom> symptoms = detector.detect(metrics);
    assertEquals(6, symptoms.size());
    SymptomsTable symptomsTable = SymptomsTable.of(symptoms);
    assertEquals(2, symptomsTable.type("POSITIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).size());
    assertEquals(2, symptomsTable.type("NEGATIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).size());
    assertEquals(1, symptomsTable.type("POSITIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).assignment("i1").size());
    assertEquals(1, symptomsTable.type("POSITIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).assignment("i3").size());
    assertEquals(1, symptomsTable.type("NEGATIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).assignment("i2").size());
    assertEquals(1, symptomsTable.type("NEGATIVE " + BaseDetector.SymptomType.SYMPTOM_PROCESSING_RATE_SKEW).assignment("i4").size());
}

Example 19 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class BackPressureDetector method detect.

/**
 * Detects all components initiating backpressure above the configured limit. Normally there
 * will be only one component
 *
 * @return A collection of symptoms each one corresponding to a components with backpressure.
 */
@Override
public Collection<Symptom> detect(Collection<Measurement> measurements) {
    publishingMetrics.executeDetectorIncr(BACK_PRESSURE_DETECTOR);
    Collection<Symptom> result = new ArrayList<>();
    Instant now = context.checkpoint();
    MeasurementsTable bpMetrics = MeasurementsTable.of(measurements).type(METRIC_BACK_PRESSURE.text());
    for (String component : bpMetrics.uniqueComponents()) {
        double compBackPressure = bpMetrics.component(component).sum();
        if (compBackPressure > noiseFilterMillis) {
            LOG.info(String.format("Detected component back-pressure for %s, total back pressure is %f", component, compBackPressure));
            List<String> addresses = Collections.singletonList(component);
            result.add(new Symptom(SYMPTOM_COMP_BACK_PRESSURE.text(), now, addresses));
        }
    }
    for (String instance : bpMetrics.uniqueInstances()) {
        double totalBP = bpMetrics.instance(instance).sum();
        if (totalBP > noiseFilterMillis) {
            LOG.info(String.format("Detected instance back-pressure for %s, total back pressure is %f", instance, totalBP));
            List<String> addresses = Collections.singletonList(instance);
            result.add(new Symptom(SYMPTOM_INSTANCE_BACK_PRESSURE.text(), now, addresses));
        }
    }
    return result;
}

Example 20 with Symptom

use of com.microsoft.dhalion.core.Symptom in project heron by twitter.

the class SkewDetector method detect.

/**
 * Detects components experiencing skew on a specific metric
 *
 * @return At most two symptoms corresponding to each affected component -- one for positive skew
 * and one for negative skew
 */
@Override
public Collection<Symptom> detect(Collection<Measurement> measurements) {
    Collection<Symptom> result = new ArrayList<>();
    MeasurementsTable metrics = MeasurementsTable.of(measurements).type(metricName);
    Instant now = context.checkpoint();
    for (String component : metrics.uniqueComponents()) {
        Set<String> addresses = new HashSet<>();
        Set<String> positiveAddresses = new HashSet<>();
        Set<String> negativeAddresses = new HashSet<>();
        double componentMax = getMaxOfAverage(metrics.component(component));
        double componentMin = getMinOfAverage(metrics.component(component));
        if (componentMax > skewRatio * componentMin) {
            // there is skew
            addresses.add(component);
            result.add(new Symptom(symptomType.text(), now, addresses));
            for (String instance : metrics.component(component).uniqueInstances()) {
                if (metrics.instance(instance).mean() >= 0.90 * componentMax) {
                    positiveAddresses.add(instance);
                }
                if (metrics.instance(instance).mean() <= 1.10 * componentMin) {
                    negativeAddresses.add(instance);
                }
            }
            if (!positiveAddresses.isEmpty()) {
                result.add(new Symptom("POSITIVE " + symptomType.text(), now, positiveAddresses));
            }
            if (!negativeAddresses.isEmpty()) {
                result.add(new Symptom("NEGATIVE " + symptomType.text(), now, negativeAddresses));
            }
        }
    }
    return result;
}