Examples with MeasurementsTable com.microsoft.dhalion.core.MeasurementsTable used on opensource projects

Example 16 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class MetricsCacheMetricsProviderTest method parsesBackPressureMetric.

@Test
public void parsesBackPressureMetric() {
    MetricsCacheMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "__time_spent_back_pressure_by_compid/container_1_split_1";
    String comp = "__stmgr__";
    TopologyManager.MetricResponse response = TopologyManager.MetricResponse.newBuilder().setStatus(Status.newBuilder().setStatus(StatusCode.OK)).addMetric(TaskMetric.newBuilder().setInstanceId("stmgr-1").addMetric(IndividualMetric.newBuilder().setName(metric).addIntervalValues(IntervalValue.newBuilder().setValue("601").setInterval(MetricInterval.newBuilder().setStart(0).setEnd(0))))).build();
    doReturn(response).when(spyMetricsProvider).getMetricsFromMetricsCache(metric, comp, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), metric, comp);
    assertEquals(1, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(1, table.component(comp).size());
    assertEquals(601, table.instance("stmgr-1").type(metric).sum(), 0.01);
}

Example 17 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class TrackerMetricsProviderTest method providesMultipleComponentMetricsFromTracker.

@Test
public void providesMultipleComponentMetricsFromTracker() {
    TrackerMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "count";
    String comp1 = "bolt-1";
    String response1 = "{\"status\": \"success\", \"executiontime\": 0.002241849899291992, " + "\"message\": \"\", \"version\": \"ver\", \"result\": " + "{\"timeline\": {\"count\": " + "{\"container_1_bolt-1_2\": {\"1497481288\": \"104\"}" + "}}, " + "\"endtime\": 1497481388, \"component\": \"bolt\", \"starttime\": 1497481208}}";
    doReturn(response1).when(spyMetricsProvider).getMetricsFromTracker(metric, comp1, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    String comp2 = "bolt-2";
    String response2 = "{\"status\": \"\", " + "\"executiontime\": 0.0026040077209472656, " + "\"message\": \"\", \"version\": \"\", " + "\"result\": {\"timeline\": {\"count\": " + "{\"container_1_bolt-2_1\": {\"1497481228\": \"12\", \"1497481348\": \"2\", " + "\"1497481168\": \"3\"}}}, " + "\"interval\": 60, \"component\": \"bolt-2\"}}";
    doReturn(response2).when(spyMetricsProvider).getMetricsFromTracker(metric, comp2, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), Collections.singletonList(metric), Arrays.asList(comp1, comp2));
    assertEquals(4, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(2, table.uniqueComponents().size());
    assertEquals(1, table.component(comp1).size());
    assertEquals(104, table.instance("container_1_bolt-1_2").sum(), 0.01);
    assertEquals(3, table.component(comp2).size());
    assertEquals(1, table.uniqueTypes().size());
    assertEquals(3, table.type(metric).instance("container_1_bolt-2_1").size());
    assertEquals(17, table.instance("container_1_bolt-2_1").sum(), 0.01);
}

Example 18 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class TrackerMetricsProviderTest method parsesBackPressureMetric.

@Test
public void parsesBackPressureMetric() {
    TrackerMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "__time_spent_back_pressure_by_compid/container_1_split_1";
    String comp = "__stmgr__";
    String response = "{\"status\": \"success\", " + "\"executiontime\": 0.30, \"message\": \"\", \"version\": \"v\", " + "\"result\": " + "{\"metrics\": {\"__time_spent_back_pressure_by_compid/container_1_split_1\": " + "{\"stmgr-1\": {\"00\" : \"601\"}}}, " + "\"interval\": 60, \"component\": \"__stmgr__\"}}";
    doReturn(response).when(spyMetricsProvider).getMetricsFromTracker(metric, comp, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), metric, comp);
    assertEquals(1, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(1, table.component(comp).size());
    assertEquals(601, table.instance("stmgr-1").type(metric).sum(), 0.01);
}

Example 19 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class BackPressureSensor method fetch.

/**
 * Computes the average (millis/sec) back-pressure caused by instances in the configured window
 *
 * @return the average value measurements
 */
@Override
public Collection<Measurement> fetch() {
    publishingMetrics.executeSensorIncr(BACKPRESSURE_SENSOR);
    Collection<Measurement> result = new ArrayList<>();
    Instant now = context.checkpoint();
    List<String> boltComponents = physicalPlanProvider.getBoltNames();
    Duration duration = getDuration();
    for (String component : boltComponents) {
        String[] boltInstanceNames = packingPlanProvider.getBoltInstanceNames(component);
        for (String instance : boltInstanceNames) {
            String metric = getMetricName() + instance;
            Collection<Measurement> stmgrResult = metricsProvider.getMeasurements(now, duration, metric, COMPONENT_STMGR);
            if (stmgrResult.isEmpty()) {
                continue;
            }
            MeasurementsTable table = MeasurementsTable.of(stmgrResult).component(COMPONENT_STMGR);
            if (table.size() == 0) {
                continue;
            }
            double averageBp = table.type(metric).sum() / duration.getSeconds();
            // The maximum value of averageBp should be 1000, i.e. 1000 millis of BP per second. Due to
            // a bug in Heron (Issue: 1753), this value could be higher in some cases. The following
            // check partially corrects the reported BP value
            averageBp = averageBp > 1000 ? 1000 : averageBp;
            Measurement measurement = new Measurement(component, instance, getMetricName(), now, averageBp);
            result.add(measurement);
        }
    }
    return result;
}

Example 20 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class BackPressureDetector method detect.

/**
 * Detects all components initiating backpressure above the configured limit. Normally there
 * will be only one component
 *
 * @return A collection of symptoms each one corresponding to a components with backpressure.
 */
@Override
public Collection<Symptom> detect(Collection<Measurement> measurements) {
    publishingMetrics.executeDetectorIncr(BACK_PRESSURE_DETECTOR);
    Collection<Symptom> result = new ArrayList<>();
    Instant now = context.checkpoint();
    MeasurementsTable bpMetrics = MeasurementsTable.of(measurements).type(METRIC_BACK_PRESSURE.text());
    for (String component : bpMetrics.uniqueComponents()) {
        double compBackPressure = bpMetrics.component(component).sum();
        if (compBackPressure > noiseFilterMillis) {
            LOG.info(String.format("Detected component back-pressure for %s, total back pressure is %f", component, compBackPressure));
            List<String> addresses = Collections.singletonList(component);
            result.add(new Symptom(SYMPTOM_COMP_BACK_PRESSURE.text(), now, addresses));
        }
    }
    for (String instance : bpMetrics.uniqueInstances()) {
        double totalBP = bpMetrics.instance(instance).sum();
        if (totalBP > noiseFilterMillis) {
            LOG.info(String.format("Detected instance back-pressure for %s, total back pressure is %f", instance, totalBP));
            List<String> addresses = Collections.singletonList(instance);
            result.add(new Symptom(SYMPTOM_INSTANCE_BACK_PRESSURE.text(), now, addresses));
        }
    }
    return result;
}