Examples with PerDatasourceShuffleMetrics org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics used on opensource projects

Example 1 with PerDatasourceShuffleMetrics

use of org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics in project druid by druid-io.

the class ShuffleMonitor method doMonitor.

@Override
public boolean doMonitor(ServiceEmitter emitter) {
    if (shuffleMetrics != null) {
        final Map<String, PerDatasourceShuffleMetrics> snapshot = shuffleMetrics.snapshotAndReset();
        snapshot.forEach((supervisorTaskId, perDatasourceShuffleMetrics) -> {
            final Builder metricBuilder = ServiceMetricEvent.builder().setDimension(SUPERVISOR_TASK_ID_DIMENSION, supervisorTaskId);
            emitter.emit(metricBuilder.build(SHUFFLE_BYTES_KEY, perDatasourceShuffleMetrics.getShuffleBytes()));
            emitter.emit(metricBuilder.build(SHUFFLE_REQUESTS_KEY, perDatasourceShuffleMetrics.getShuffleRequests()));
        });
    }
    return true;
}

Example 2 with PerDatasourceShuffleMetrics

use of org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics in project druid by druid-io.

the class ShuffleMetricsTest method testSnapshotUnmodifiable.

@Test
public void testSnapshotUnmodifiable() {
    expectedException.expect(UnsupportedOperationException.class);
    new ShuffleMetrics().snapshotAndReset().put("k", new PerDatasourceShuffleMetrics());
}

Example 3 with PerDatasourceShuffleMetrics

use of org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics in project druid by druid-io.

the class ShuffleMonitorTest method testDoMonitor.

@Test
public void testDoMonitor() {
    final ShuffleMetrics shuffleMetrics = Mockito.mock(ShuffleMetrics.class);
    final PerDatasourceShuffleMetrics perDatasourceShuffleMetrics = new PerDatasourceShuffleMetrics();
    perDatasourceShuffleMetrics.accumulate(100);
    perDatasourceShuffleMetrics.accumulate(200);
    perDatasourceShuffleMetrics.accumulate(10);
    Mockito.when(shuffleMetrics.snapshotAndReset()).thenReturn(ImmutableMap.of("supervisor", perDatasourceShuffleMetrics));
    final StubServiceEmitter emitter = new StubServiceEmitter("service", "host");
    final ShuffleMonitor monitor = new ShuffleMonitor();
    monitor.setShuffleMetrics(shuffleMetrics);
    Assert.assertTrue(monitor.doMonitor(emitter));
    final List<Event> events = emitter.getEvents();
    Assert.assertEquals(2, events.size());
    Assert.assertSame(ServiceMetricEvent.class, events.get(0).getClass());
    ServiceMetricEvent event = (ServiceMetricEvent) events.get(0);
    Assert.assertEquals(ShuffleMonitor.SHUFFLE_BYTES_KEY, event.getMetric());
    Assert.assertEquals(310L, event.getValue());
    Assert.assertEquals(ImmutableMap.of(ShuffleMonitor.SUPERVISOR_TASK_ID_DIMENSION, "supervisor"), event.getUserDims());
    Assert.assertSame(ServiceMetricEvent.class, events.get(1).getClass());
    event = (ServiceMetricEvent) events.get(1);
    Assert.assertEquals(ShuffleMonitor.SHUFFLE_REQUESTS_KEY, event.getMetric());
    Assert.assertEquals(3, event.getValue());
    Assert.assertEquals(ImmutableMap.of(ShuffleMonitor.SUPERVISOR_TASK_ID_DIMENSION, "supervisor"), event.getUserDims());
}

Example 4 with PerDatasourceShuffleMetrics

use of org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics in project druid by druid-io.

the class ShuffleResourceTest method testGetPartitionWithValidParamsReturnOk.

@Test
public void testGetPartitionWithValidParamsReturnOk() throws IOException {
    final String supervisorTaskId = "supervisorTask";
    final String subtaskId = "subtaskId";
    final Interval interval = Intervals.of("2020-01-01/P1D");
    final DataSegment segment = newSegment(interval);
    final File segmentDir = generateSegmentDir("test");
    intermediaryDataManager.addSegment(supervisorTaskId, subtaskId, segment, segmentDir);
    final Response response = shuffleResource.getPartition(supervisorTaskId, subtaskId, interval.getStart().toString(), interval.getEnd().toString(), segment.getId().getPartitionNum());
    final Map<String, PerDatasourceShuffleMetrics> snapshot = shuffleMetrics.snapshotAndReset();
    Assert.assertEquals(Status.OK.getStatusCode(), response.getStatus());
    Assert.assertEquals(1, snapshot.get(supervisorTaskId).getShuffleRequests());
    Assert.assertEquals(254, snapshot.get(supervisorTaskId).getShuffleBytes());
}

Example 5 with PerDatasourceShuffleMetrics

use of org.apache.druid.indexing.worker.shuffle.ShuffleMetrics.PerDatasourceShuffleMetrics in project druid by druid-io.

the class ShuffleMetricsTest method testShuffleRequested.

@Test
public void testShuffleRequested() {
    final ShuffleMetrics metrics = new ShuffleMetrics();
    final String supervisorTask1 = "supervisor1";
    final String supervisorTask2 = "supervisor2";
    final String supervisorTask3 = "supervisor3";
    metrics.shuffleRequested(supervisorTask1, 1024);
    metrics.shuffleRequested(supervisorTask2, 10);
    metrics.shuffleRequested(supervisorTask1, 512);
    metrics.shuffleRequested(supervisorTask3, 10000);
    metrics.shuffleRequested(supervisorTask2, 30);
    final Map<String, PerDatasourceShuffleMetrics> snapshot = metrics.snapshotAndReset();
    Assert.assertEquals(ImmutableSet.of(supervisorTask1, supervisorTask2, supervisorTask3), snapshot.keySet());
    PerDatasourceShuffleMetrics perDatasourceShuffleMetrics = snapshot.get(supervisorTask1);
    Assert.assertEquals(2, perDatasourceShuffleMetrics.getShuffleRequests());
    Assert.assertEquals(1536, perDatasourceShuffleMetrics.getShuffleBytes());
    perDatasourceShuffleMetrics = snapshot.get(supervisorTask2);
    Assert.assertEquals(2, perDatasourceShuffleMetrics.getShuffleRequests());
    Assert.assertEquals(40, perDatasourceShuffleMetrics.getShuffleBytes());
    perDatasourceShuffleMetrics = snapshot.get(supervisorTask3);
    Assert.assertEquals(1, perDatasourceShuffleMetrics.getShuffleRequests());
    Assert.assertEquals(10000, perDatasourceShuffleMetrics.getShuffleBytes());
}