Examples with MetricsInfo org.apache.heron.spi.metricsmgr.metrics.MetricsInfo used on opensource projects

Example 6 with MetricsInfo

use of org.apache.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class MetricsManagerServer method handlePublisherPublishMessage.

private void handlePublisherPublishMessage(Metrics.MetricPublisher request, Metrics.MetricPublisherPublishMessage message) {
    if (message.getMetricsCount() <= 0 && message.getExceptionsCount() <= 0) {
        LOG.log(Level.SEVERE, "Publish message has no metrics nor exceptions for message from hostname: {0}," + " component_name: {1}, port: {2}, instance_id: {3}, instance_index: {4}", new Object[] { request.getHostname(), request.getComponentName(), request.getPort(), request.getInstanceId(), request.getInstanceIndex() });
        return;
    }
    // Convert the message to MetricsRecord
    String source = MetricsUtil.createSource(request.getHostname(), request.getPort(), request.getComponentName(), request.getInstanceId());
    List<MetricsInfo> metricsInfos = new ArrayList<MetricsInfo>(message.getMetricsCount());
    for (Metrics.MetricDatum metricDatum : message.getMetricsList()) {
        MetricsInfo info = new MetricsInfo(metricDatum.getName(), metricDatum.getValue());
        metricsInfos.add(info);
    }
    List<ExceptionInfo> exceptionInfos = new ArrayList<ExceptionInfo>(message.getExceptionsCount());
    for (Metrics.ExceptionData exceptionData : message.getExceptionsList()) {
        ExceptionInfo exceptionInfo = new ExceptionInfo(exceptionData.getStacktrace(), exceptionData.getLasttime(), exceptionData.getFirsttime(), exceptionData.getCount(), exceptionData.getLogging());
        exceptionInfos.add(exceptionInfo);
    }
    LOG.info(String.format("%d MetricsInfo and %d ExceptionInfo to push", metricsInfos.size(), exceptionInfos.size()));
    // Update the metrics
    serverMetricsCounters.scope(SERVER_METRICS_RECEIVED).incrBy(metricsInfos.size());
    serverMetricsCounters.scope(SERVER_EXCEPTIONS_RECEIVED).incrBy(exceptionInfos.size());
    MetricsRecord record = new MetricsRecord(source, metricsInfos, exceptionInfos);
    // Push MetricsRecord to Communicator, which would wake up ExecutorLooper bind with IMetricsSink
    synchronized (metricsSinkCommunicators) {
        Iterator<String> itr = metricsSinkCommunicators.keySet().iterator();
        while (itr.hasNext()) {
            String key = itr.next();
            Communicator<MetricsRecord> c = metricsSinkCommunicators.get(key);
            c.offer(record);
            serverMetricsCounters.scope(SERVER_COMMUNICATOR_OFFER).incr();
            serverMetricsCounters.scope(SERVER_COMMUNICATOR_SIZE + "-" + key).incrBy(c.size());
        }
    }
}

Example 7 with MetricsInfo

use of org.apache.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class TManagerSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    // Format it into TopologyManager.PublishMetrics
    // The format of record is "host:port/componentName/instanceId"
    // So MetricsRecord.getSource().split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = MetricsUtil.splitRecordSource(record);
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyManager.PublishMetrics.Builder publishMetrics = TopologyManager.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tManagerMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyManager.MetricDatum metricDatum = TopologyManager.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyManager.TmanagerExceptionLog exceptionLog = TopologyManager.TmanagerExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
    checkCommunicator(metricsCommunicator, MAX_COMMUNICATOR_SIZE);
}

Example 8 with MetricsInfo

use of org.apache.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class WebSinkTest method before.

@Before
public void before() throws IOException {
    defaultConf = new HashMap<>();
    defaultConf.put("port", "9999");
    defaultConf.put("path", "test");
    defaultConf.put("flat-metrics", "true");
    defaultConf.put("include-topology-name", "false");
    context = Mockito.mock(SinkContext.class);
    Mockito.when(context.getTopologyName()).thenReturn("testTopology");
    Mockito.when(context.getSinkId()).thenReturn("testId");
    Iterable<MetricsInfo> infos = Arrays.asList(new MetricsInfo("metric_1", "1.0"), new MetricsInfo("metric_2", "2.0"));
    records = Arrays.asList(new MetricsRecord("machine/stuff/record_1", infos, Collections.<ExceptionInfo>emptyList()), new MetricsRecord("record_2", infos, Collections.<ExceptionInfo>emptyList()));
}

Example 9 with MetricsInfo

use of org.apache.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class FileSink method convertRecordToJSON.

private String convertRecordToJSON(MetricsRecord record) {
    int metricsCount = 0;
    int exceptionsCount = 0;
    // Pack metrics as a map
    Map<String, String> metrics = new HashMap<String, String>();
    for (MetricsInfo metricsInfo : record.getMetrics()) {
        metrics.put(metricsInfo.getName(), metricsInfo.getValue());
        metricsCount++;
    }
    // Pack exceptions as a list of map
    LinkedList<Object> exceptions = new LinkedList<Object>();
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        Map<String, Object> exception = new HashMap<String, Object>();
        exception.put("firstTime", exceptionInfo.getFirstTime());
        exception.put("lastTime", exceptionInfo.getLastTime());
        exception.put("logging", exceptionInfo.getLogging());
        exception.put("stackTrace", exceptionInfo.getStackTrace());
        exception.put("count", exceptionInfo.getCount());
        exceptions.add(exception);
        exceptionsCount++;
    }
    // Pack the whole MetricsRecord as a map
    Map<String, Object> jsonToWrite = new HashMap<String, Object>();
    jsonToWrite.put("timestamp", record.getTimestamp());
    jsonToWrite.put("source", record.getSource());
    jsonToWrite.put("context", record.getContext());
    jsonToWrite.put("metrics", metrics);
    jsonToWrite.put("exceptions", exceptions);
    // Update metrics
    sinkContext.exportCountMetric(METRICS_COUNT, metricsCount);
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, exceptionsCount);
    String result = "";
    try {
        result = MAPPER.writeValueAsString(jsonToWrite);
    } catch (JsonProcessingException e) {
        LOG.log(Level.SEVERE, "Could not convert map to JSONString: " + jsonToWrite.toString(), e);
    }
    return result;
}

Example 10 with MetricsInfo

use of org.apache.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class MetricsManagerServerTest method testMetricsManagerServer.

/**
 * Method: addSinkCommunicator(Communicator<MetricsRecord> communicator)
 */
@Test
public void testMetricsManagerServer() throws InterruptedException {
    String name = "test_communicator";
    CountDownLatch offersLatch = new CountDownLatch(MESSAGE_SIZE);
    Communicator<MetricsRecord> sinkCommunicator = CommunicatorTestHelper.spyCommunicator(new Communicator<MetricsRecord>(), offersLatch);
    metricsManagerServer.addSinkCommunicator(name, sinkCommunicator);
    serverTester.start();
    HeronServerTester.await(offersLatch);
    int messages = 0;
    while (!sinkCommunicator.isEmpty()) {
        int exceptions = 0;
        int metrics = 0;
        MetricsRecord record = sinkCommunicator.poll();
        Assert.assertEquals("hostname:0/component/instance-id", record.getSource());
        Assert.assertEquals("default", record.getContext());
        for (MetricsInfo info : record.getMetrics()) {
            Assert.assertEquals(METRIC_NAME, info.getName());
            Assert.assertEquals(METRIC_VALUE, info.getValue());
            metrics++;
        }
        Assert.assertEquals(METRICS_COUNT, metrics);
        for (ExceptionInfo info : record.getExceptions()) {
            Assert.assertEquals(STACK_TRACE, info.getStackTrace());
            Assert.assertEquals(LAST_TIME, info.getLastTime());
            Assert.assertEquals(FIRST_TIME, info.getFirstTime());
            Assert.assertEquals(LOGGING, info.getLogging());
            Assert.assertEquals(EXCEPTION_COUNT, info.getCount());
            exceptions++;
        }
        Assert.assertEquals(METRICS_COUNT, exceptions);
        messages++;
    }
    Assert.assertEquals(MESSAGE_SIZE, messages);
}