Examples with MeasurementsTable com.microsoft.dhalion.core.MeasurementsTable used on opensource projects

Example 11 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class ExecuteCountSensorTest method providesBoltExecutionCountMetrics.

@Test
public void providesBoltExecutionCountMetrics() {
    Instant now = Instant.now();
    String metric = METRIC_EXE_COUNT.text();
    PhysicalPlanProvider topologyProvider = mock(PhysicalPlanProvider.class);
    when(topologyProvider.getBoltNames()).thenReturn(Arrays.asList(new String[] { "bolt-1", "bolt-2" }));
    MetricsProvider metricsProvider = mock(MetricsProvider.class);
    Collection<Measurement> result = new ArrayList<>();
    result.add(new Measurement("bolt-1", "container_1_bolt-1_1", metric, now, 123));
    result.add(new Measurement("bolt-1", "container_1_bolt-1_2", metric, now, 345));
    result.add(new Measurement("bolt-2", "container_1_bolt-2_3", metric, now, 321));
    result.add(new Measurement("bolt-2", "container_1_bolt-2_4", metric, now, 543));
    Collection<String> comps = Arrays.asList("bolt-1", "bolt-2");
    when(metricsProvider.getMeasurements(any(Instant.class), eq(DEFAULT_METRIC_DURATION), eq(Collections.singletonList(metric)), eq(comps))).thenReturn(result);
    ExecuteCountSensor executeCountSensor = new ExecuteCountSensor(topologyProvider, null, metricsProvider);
    PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
    when(context.checkpoint()).thenReturn(now);
    executeCountSensor.initialize(context);
    Collection<Measurement> componentMetrics = executeCountSensor.fetch();
    assertEquals(4, componentMetrics.size());
    MeasurementsTable table = MeasurementsTable.of(componentMetrics);
    assertEquals(123, table.component("bolt-1").instance("container_1_bolt-1_1").type(metric).sum(), 0.01);
    assertEquals(345, table.component("bolt-1").instance("container_1_bolt-1_2").type(metric).sum(), 0.01);
    assertEquals(321, table.component("bolt-2").instance("container_1_bolt-2_3").type(metric).sum(), 0.01);
    assertEquals(543, table.component("bolt-2").instance("container_1_bolt-2_4").type(metric).sum(), 0.01);
}

Example 12 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class TrackerMetricsProviderTest method provides1Comp2InstanceMetricsFromTracker.

@Test
public void provides1Comp2InstanceMetricsFromTracker() {
    TrackerMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "count";
    String comp = "bolt";
    String response = "{\"status\": \"success\", \"executiontime\": 0.002241849899291992, " + "\"message\": \"\", \"version\": \"ver\", \"result\": " + "{\"timeline\": {\"count\": " + "{\"container_1_bolt_1\": {\"1497481288\": \"104\"}, " + "\"container_1_bolt_2\": {\"1497481228\": \"12\", \"1497481348\": \"2\", " + "\"1497481168\": \"3\"}}}, " + "\"endtime\": 1497481388, \"component\": \"bolt\", \"starttime\": 1497481208}}";
    doReturn(response).when(spyMetricsProvider).getMetricsFromTracker(metric, comp, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), metric, comp);
    assertEquals(4, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(4, table.component(comp).size());
    assertEquals(2, table.uniqueInstances().size());
    assertEquals(1, table.uniqueTypes().size());
    assertEquals(1, table.instance("container_1_bolt_1").size());
    assertEquals(104, table.instance("container_1_bolt_1").sum(), 0.01);
    assertEquals(3, table.instance("container_1_bolt_2").size());
    assertEquals(17, table.instance("container_1_bolt_2").sum(), 0.01);
}

Example 13 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class TrackerMetricsProviderTest method testGetTimeLineMetrics.

@Test
public void testGetTimeLineMetrics() {
    TrackerMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "count";
    String comp = "bolt";
    String response = "{\"status\": \"success\", \"executiontime\": 0.002241849899291992, " + "\"message\": \"\", \"version\": \"ver\", \"result\": " + "{\"timeline\": {\"count\": " + "{\"container_1_bolt_1\": {\"1497481288\": \"104\"}, " + "\"container_1_bolt_2\": {\"1497481228\": \"12\", \"1497481348\": \"2\", " + "\"1497481168\": \"3\"}}}, " + "\"endtime\": 1497481388, \"component\": \"bolt\", \"starttime\": 1497481208}}";
    doReturn(response).when(spyMetricsProvider).getMetricsFromTracker(metric, comp, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), metric, comp);
    assertEquals(4, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(4, table.component(comp).size());
    MeasurementsTable result = table.instance("container_1_bolt_1");
    assertEquals(1, result.size());
    assertEquals(104, result.instant(Instant.ofEpochSecond(1497481288)).sum(), 0.01);
    result = table.instance("container_1_bolt_2");
    assertEquals(3, result.size());
    assertEquals(12, result.instant(Instant.ofEpochSecond(1497481228L)).sum(), 0.01);
    assertEquals(2, result.instant(Instant.ofEpochSecond(1497481348L)).sum(), 0.01);
    assertEquals(3, result.instant(Instant.ofEpochSecond(1497481168L)).sum(), 0.01);
}

Example 14 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class ScaleUpResolver method computeScaleUpFactor.

@VisibleForTesting
int computeScaleUpFactor(String compName) {
    Instant newest = context.checkpoint();
    Instant oldest = context.previousCheckpoint();
    MeasurementsTable table = context.measurements().component(compName).type(METRIC_BACK_PRESSURE.text()).between(oldest, newest);
    double totalCompBpTime = table.sum();
    LOG.info(String.format("Component: %s, bpTime: %.0f", compName, totalCompBpTime));
    if (totalCompBpTime >= 1000) {
        totalCompBpTime = 999;
        LOG.warning(String.format("Comp:%s, bpTime after filter: %.0f", compName, totalCompBpTime));
    }
    double unusedCapacity = (1.0 * totalCompBpTime) / (1000 - totalCompBpTime);
    // scale up fencing: do not scale more than 4 times the current size
    unusedCapacity = unusedCapacity > 4.0 ? 4.0 : unusedCapacity;
    int parallelism = (int) Math.ceil(table.uniqueInstances().size() * (1 + unusedCapacity));
    LOG.info(String.format("Component's, %s, unused capacity is: %.3f. New parallelism: %d", compName, unusedCapacity, parallelism));
    return parallelism;
}

Example 15 with MeasurementsTable

use of com.microsoft.dhalion.core.MeasurementsTable in project heron by twitter.

the class MetricsCacheMetricsProviderTest method providesMultipleComponentMetricsFromMetricsCache.

@Test
public void providesMultipleComponentMetricsFromMetricsCache() {
    MetricsCacheMetricsProvider spyMetricsProvider = createMetricsProviderSpy();
    String metric = "count";
    String comp1 = "bolt-1";
    TopologyManager.MetricResponse response1 = TopologyManager.MetricResponse.newBuilder().setStatus(Status.newBuilder().setStatus(StatusCode.OK)).addMetric(TaskMetric.newBuilder().setInstanceId("container_1_bolt-1_2").addMetric(IndividualMetric.newBuilder().setName(metric).addIntervalValues(IntervalValue.newBuilder().setValue("104").setInterval(MetricInterval.newBuilder().setStart(1497481288).setEnd(1497481288))))).build();
    doReturn(response1).when(spyMetricsProvider).getMetricsFromMetricsCache(metric, comp1, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    String comp2 = "bolt-2";
    TopologyManager.MetricResponse response2 = TopologyManager.MetricResponse.newBuilder().setStatus(Status.newBuilder().setStatus(StatusCode.OK)).addMetric(TaskMetric.newBuilder().setInstanceId("container_1_bolt-2_1").addMetric(IndividualMetric.newBuilder().setName(metric).addIntervalValues(IntervalValue.newBuilder().setValue("12").setInterval(MetricInterval.newBuilder().setStart(1497481228).setEnd(1497481228))).addIntervalValues(IntervalValue.newBuilder().setValue("2").setInterval(MetricInterval.newBuilder().setStart(1497481348).setEnd(1497481348))).addIntervalValues(IntervalValue.newBuilder().setValue("3").setInterval(MetricInterval.newBuilder().setStart(1497481168).setEnd(1497481168))))).build();
    doReturn(response2).when(spyMetricsProvider).getMetricsFromMetricsCache(metric, comp2, Instant.ofEpochSecond(10), Duration.ofSeconds(60));
    Collection<Measurement> metrics = spyMetricsProvider.getMeasurements(Instant.ofEpochSecond(10), Duration.ofSeconds(60), Collections.singletonList(metric), Arrays.asList(comp1, comp2));
    assertEquals(4, metrics.size());
    MeasurementsTable table = MeasurementsTable.of(metrics);
    assertEquals(2, table.uniqueComponents().size());
    assertEquals(1, table.component(comp1).size());
    assertEquals(104, table.instance("container_1_bolt-1_2").sum(), 0.01);
    assertEquals(3, table.component(comp2).size());
    assertEquals(1, table.uniqueTypes().size());
    assertEquals(3, table.type(metric).instance("container_1_bolt-2_1").size());
    assertEquals(17, table.instance("container_1_bolt-2_1").sum(), 0.01);
}