Examples with ServiceMetricEvent org.apache.druid.java.util.emitter.service.ServiceMetricEvent used on opensource projects

Example 46 with ServiceMetricEvent

use of org.apache.druid.java.util.emitter.service.ServiceMetricEvent in project druid by apache.

the class KafkaEmitterTest method testKafkaEmitter.

// there is 1 seconds wait in kafka emitter before it starts sending events to broker, set a timeout for 5 seconds
@Test(timeout = 5_000)
public void testKafkaEmitter() throws InterruptedException {
    final List<ServiceMetricEvent> serviceMetricEvents = ImmutableList.of(ServiceMetricEvent.builder().build("m1", 1).build("service", "host"));
    final List<AlertEvent> alertEvents = ImmutableList.of(new AlertEvent("service", "host", "description"));
    final List<RequestLogEvent> requestLogEvents = ImmutableList.of(DefaultRequestLogEventBuilderFactory.instance().createRequestLogEventBuilder("requests", RequestLogLine.forSql("", null, DateTimes.nowUtc(), null, new QueryStats(ImmutableMap.of()))).build("service", "host"));
    int totalEvents = serviceMetricEvents.size() + alertEvents.size() + requestLogEvents.size();
    int totalEventsExcludingRequestLogEvents = totalEvents - requestLogEvents.size();
    final CountDownLatch countDownSentEvents = new CountDownLatch(requestTopic == null ? totalEventsExcludingRequestLogEvents : totalEvents);
    final KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    final KafkaEmitter kafkaEmitter = new KafkaEmitter(new KafkaEmitterConfig("", "metrics", "alerts", requestTopic, "test-cluster", null), new ObjectMapper()) {

        @Override
        protected Producer<String, String> setKafkaProducer() {
            // override send interval to 1 second
            sendInterval = 1;
            return producer;
        }
    };
    when(producer.send(any(), any())).then((invocation) -> {
        countDownSentEvents.countDown();
        return null;
    });
    kafkaEmitter.start();
    for (Event event : serviceMetricEvents) {
        kafkaEmitter.emit(event);
    }
    for (Event event : alertEvents) {
        kafkaEmitter.emit(event);
    }
    for (Event event : requestLogEvents) {
        kafkaEmitter.emit(event);
    }
    countDownSentEvents.await();
    Assert.assertEquals(0, kafkaEmitter.getMetricLostCount());
    Assert.assertEquals(0, kafkaEmitter.getAlertLostCount());
    Assert.assertEquals(requestTopic == null ? requestLogEvents.size() : 0, kafkaEmitter.getRequestLostCount());
    Assert.assertEquals(0, kafkaEmitter.getInvalidLostCount());
}

Example 47 with ServiceMetricEvent

use of org.apache.druid.java.util.emitter.service.ServiceMetricEvent in project druid by druid-io.

the class EmitClusterStatsAndMetricsTest method testRunOnlyEmitStatsForHistoricalDuties.

@Test
public void testRunOnlyEmitStatsForHistoricalDuties() {
    ArgumentCaptor<ServiceEventBuilder> argumentCaptor = ArgumentCaptor.forClass(ServiceEventBuilder.class);
    Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter);
    Mockito.when(mockDruidCoordinatorRuntimeParams.getCoordinatorStats()).thenReturn(mockCoordinatorStats);
    Mockito.when(mockDruidCoordinatorRuntimeParams.getDruidCluster()).thenReturn(mockDruidCluster);
    Mockito.when(mockDruidCoordinatorRuntimeParams.getDatabaseRuleManager()).thenReturn(mockMetadataRuleManager);
    Mockito.when(mockDruidCoordinator.computeNumsUnavailableUsedSegmentsPerDataSource()).thenReturn(Object2IntMaps.emptyMap());
    Mockito.when(mockDruidCoordinator.computeUnderReplicationCountsPerDataSourcePerTier()).thenReturn(ImmutableMap.of());
    CoordinatorDuty duty = new EmitClusterStatsAndMetrics(mockDruidCoordinator, DruidCoordinator.HISTORICAL_MANAGEMENT_DUTIES_DUTY_GROUP, false);
    duty.run(mockDruidCoordinatorRuntimeParams);
    Mockito.verify(mockServiceEmitter, Mockito.atLeastOnce()).emit(argumentCaptor.capture());
    List<ServiceEventBuilder> emittedEvents = argumentCaptor.getAllValues();
    boolean foundCompactMetric = false;
    boolean foundHistoricalDutyMetric = false;
    for (ServiceEventBuilder eventBuilder : emittedEvents) {
        ServiceMetricEvent serviceMetricEvent = ((ServiceMetricEvent) eventBuilder.build("x", "x"));
        String metric = serviceMetricEvent.getMetric();
        if ("segment/overShadowed/count".equals(metric)) {
            foundHistoricalDutyMetric = true;
        } else if ("compact/task/count".equals(metric)) {
            foundCompactMetric = true;
        }
        String dutyGroup = (String) serviceMetricEvent.getUserDims().get("dutyGroup");
        Assert.assertNotNull(dutyGroup);
        Assert.assertEquals(DruidCoordinator.HISTORICAL_MANAGEMENT_DUTIES_DUTY_GROUP, dutyGroup);
    }
    Assert.assertTrue(foundHistoricalDutyMetric);
    Assert.assertFalse(foundCompactMetric);
}

Example 48 with ServiceMetricEvent

use of org.apache.druid.java.util.emitter.service.ServiceMetricEvent in project druid by druid-io.

the class JettyServerModuleTest method testJettyServerModule.

@Test
public void testJettyServerModule() {
    List<Event> events = new ArrayList<>();
    ServiceEmitter serviceEmitter = new ServiceEmitter("service", "host", Mockito.mock(Emitter.class)) {

        @Override
        public void emit(Event event) {
            events.add(event);
        }
    };
    QueuedThreadPool jettyServerThreadPool = Mockito.mock(QueuedThreadPool.class);
    JettyServerModule.setJettyServerThreadPool(jettyServerThreadPool);
    Mockito.when(jettyServerThreadPool.getThreads()).thenReturn(100);
    Mockito.when(jettyServerThreadPool.getIdleThreads()).thenReturn(40);
    Mockito.when(jettyServerThreadPool.isLowOnThreads()).thenReturn(true);
    Mockito.when(jettyServerThreadPool.getMinThreads()).thenReturn(30);
    Mockito.when(jettyServerThreadPool.getMaxThreads()).thenReturn(100);
    Mockito.when(jettyServerThreadPool.getQueueSize()).thenReturn(50);
    Mockito.when(jettyServerThreadPool.getBusyThreads()).thenReturn(60);
    JettyServerModule.JettyMonitor jettyMonitor = new JettyServerModule.JettyMonitor("ds", "t0");
    jettyMonitor.doMonitor(serviceEmitter);
    Assert.assertEquals(8, events.size());
    List<Pair<String, Number>> expectedEvents = Arrays.asList(new Pair<>("jetty/numOpenConnections", 0), new Pair<>("jetty/threadPool/total", 100), new Pair<>("jetty/threadPool/idle", 40), new Pair<>("jetty/threadPool/isLowOnThreads", 1), new Pair<>("jetty/threadPool/min", 30), new Pair<>("jetty/threadPool/max", 100), new Pair<>("jetty/threadPool/queueSize", 50), new Pair<>("jetty/threadPool/busy", 60));
    for (int i = 0; i < expectedEvents.size(); i++) {
        Pair<String, Number> expected = expectedEvents.get(i);
        ServiceMetricEvent actual = (ServiceMetricEvent) (events.get(i));
        Assert.assertEquals(expected.lhs, actual.getMetric());
        Assert.assertEquals(expected.rhs, actual.getValue());
    }
}

Example 49 with ServiceMetricEvent

use of org.apache.druid.java.util.emitter.service.ServiceMetricEvent in project druid by druid-io.

the class MetricsEmittingQueryProcessingPoolTest method testPrioritizedExecutorDelegate.

@Test
public void testPrioritizedExecutorDelegate() {
    PrioritizedExecutorService service = Mockito.mock(PrioritizedExecutorService.class);
    Mockito.when(service.getQueueSize()).thenReturn(10);
    ExecutorServiceMonitor monitor = new ExecutorServiceMonitor();
    List<Event> events = new ArrayList<>();
    MetricsEmittingQueryProcessingPool processingPool = new MetricsEmittingQueryProcessingPool(service, monitor);
    Assert.assertSame(service, processingPool.delegate());
    ServiceEmitter serviceEmitter = new ServiceEmitter("service", "host", Mockito.mock(Emitter.class)) {

        @Override
        public void emit(Event event) {
            events.add(event);
        }
    };
    monitor.doMonitor(serviceEmitter);
    Assert.assertEquals(1, events.size());
    Assert.assertEquals(((ServiceMetricEvent) (events.get(0))).getMetric(), "segment/scan/pending");
    Assert.assertEquals(((ServiceMetricEvent) (events.get(0))).getValue(), 10);
}

Example 50 with ServiceMetricEvent

use of org.apache.druid.java.util.emitter.service.ServiceMetricEvent in project druid by druid-io.

the class InfluxdbEmitterTest method testMetricIsInDefaultDimensionWhitelist.

@Test
public void testMetricIsInDefaultDimensionWhitelist() {
    DateTime date = new DateTime(2017, 10, 30, 10, 00, // 10:00am on 30/10/2017 = 1509357600000000000 in epoch nanoseconds
    DateTimeZone.UTC);
    String metric = "metric/time";
    Number value = 1234;
    ImmutableMap<String, String> serviceDims = ImmutableMap.of("service", "druid/historical", "host", "localhost", "version", "0.10.0");
    ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder();
    ServiceEventBuilder eventBuilder = builder.build(date, metric, value);
    builder.setDimension("dataSource", "wikipedia");
    builder.setDimension("taskType", "index");
    ServiceMetricEvent event = (ServiceMetricEvent) eventBuilder.build(serviceDims);
    InfluxdbEmitterConfig config = new InfluxdbEmitterConfig("localhost", 8086, null, null, null, null, "dbname", 10000, 15000, 30000, "adam", "password", null);
    InfluxdbEmitter influxdbEmitter = new InfluxdbEmitter(config);
    String expected = "druid_metric,service=druid/historical,hostname=localhost,dataSource=wikipedia,taskType=index druid_time=1234 1509357600000000000" + "\n";
    String actual = influxdbEmitter.transformForInfluxSystems(event);
    Assert.assertEquals(expected, actual);
}