Examples with MetricsInfo com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo used on opensource projects

Example 1 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class WebSinkTest method testGroupedMetrics.

/**
   * Testing grouped map with metrics
   */
@Test
public void testGroupedMetrics() {
    Map<String, Object> conf = new HashMap<>(defaultConf);
    conf.put("flat-metrics", "false");
    WebTestSink sink = new WebTestSink();
    sink.init(conf, context);
    for (MetricsRecord r : records) {
        sink.processRecord(r);
    }
    //Update and override MetricsRecord 1
    Iterable<MetricsInfo> infos2 = Arrays.asList(new MetricsInfo("metric_1", "3.0"), new MetricsInfo("metric_3", "1.0"));
    sink.processRecord(new MetricsRecord(records.get(0).getSource(), infos2, Collections.<ExceptionInfo>emptyList()));
    Map<String, Object> results = sink.getMetrics();
    Assert.assertEquals(2, results.size());
    @SuppressWarnings("unchecked") Map<String, Object> record1 = (Map<String, Object>) results.get("/stuff/record_1");
    @SuppressWarnings("unchecked") Map<String, Object> record2 = (Map<String, Object>) results.get("/record_2");
    Assert.assertEquals(record1.get("metric_1"), 3.0d);
    Assert.assertEquals(record1.get("metric_2"), 2.0d);
    Assert.assertEquals(record1.get("metric_3"), 1.0d);
    Assert.assertEquals(record2.get("metric_1"), 1.0d);
    Assert.assertEquals(record2.get("metric_2"), 2.0d);
}

Example 2 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class GraphiteSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    StringBuilder lines = new StringBuilder();
    StringBuilder metricsPathPrefix = new StringBuilder();
    // Configure the hierarchical place to display the graph.
    // The metricsPathPrefix would look like:
    // {metricsPrefix}.{topologyName}.{host:port/componentName/instanceId}
    metricsPathPrefix.append(metricsPrefix).append(".").append(topologyName).append(".").append(record.getSource());
    // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds.
    long timestamp = record.getTimestamp() / 1000L;
    //    {metricValue} {timestamp} \n
    for (MetricsInfo metricsInfo : record.getMetrics()) {
        lines.append(metricsPathPrefix.toString() + "." + metricsInfo.getName().replace(' ', '.')).append(" ").append(metricsInfo.getValue()).append(" ").append(timestamp).append("\n");
    }
    try {
        graphite.write(lines.toString());
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Error sending metrics to Graphite. Dropping messages...", e);
        // 2. GraphiteSink.close() would contain more cleaning logic for the whole sink
        try {
            graphite.close();
        } catch (IOException e1) {
            LOG.log(Level.SEVERE, "Error closing connection to Graphite", e1);
        }
    }
}

Example 3 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class TMasterSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    // Format it into TopologyMaster.PublishMetrics
    // The format of source is "host:port/componentName/instanceId"
    // So source.split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = record.getSource().split("/");
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyMaster.PublishMetrics.Builder publishMetrics = TopologyMaster.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tMasterMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyMaster.MetricDatum metricDatum = TopologyMaster.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyMaster.TmasterExceptionLog exceptionLog = TopologyMaster.TmasterExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
}

Example 4 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class MetricsManagerServerTest method testMetricsManagerServer.

/**
   * Method: addSinkCommunicator(Communicator<MetricsRecord> communicator)
   */
@Test
public void testMetricsManagerServer() throws Exception {
    final Communicator<MetricsRecord> sinkCommunicator = new Communicator<MetricsRecord>();
    metricsManagerServer.addSinkCommunicator(sinkCommunicator);
    // First run Server
    runServer();
    // Wait a while for server fully starting
    Thread.sleep(3 * 1000);
    // Then run Client
    runClient();
    // Wait some while to let message fully send out
    Thread.sleep(10 * 1000);
    int messages = 0;
    while (!sinkCommunicator.isEmpty()) {
        int exceptions = 0;
        int metrics = 0;
        MetricsRecord record = sinkCommunicator.poll();
        Assert.assertEquals("hostname:0/component/instance-id", record.getSource());
        Assert.assertEquals("default", record.getContext());
        for (MetricsInfo info : record.getMetrics()) {
            Assert.assertEquals(METRIC_NAME, info.getName());
            Assert.assertEquals(METRIC_VALUE, info.getValue());
            metrics++;
        }
        Assert.assertEquals(N, metrics);
        for (ExceptionInfo info : record.getExceptions()) {
            Assert.assertEquals(STACK_TRACE, info.getStackTrace());
            Assert.assertEquals(LAST_TIME, info.getLastTime());
            Assert.assertEquals(FIRST_TIME, info.getFirstTime());
            Assert.assertEquals(LOGGING, info.getLogging());
            Assert.assertEquals(EXCEPTION_COUNT, info.getCount());
            exceptions++;
        }
        Assert.assertEquals(N, exceptions);
        messages++;
    }
    Assert.assertEquals(MESSAGE_SIZE, messages);
}

Example 5 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class WebSinkTest method before.

@Before
public void before() throws IOException {
    defaultConf = new HashMap<>();
    defaultConf.put("port", "9999");
    defaultConf.put("path", "test");
    defaultConf.put("flat-metrics", "true");
    defaultConf.put("include-topology-name", "false");
    context = Mockito.mock(SinkContext.class);
    Mockito.when(context.getTopologyName()).thenReturn("testTopology");
    Mockito.when(context.getSinkId()).thenReturn("testId");
    Iterable<MetricsInfo> infos = Arrays.asList(new MetricsInfo("metric_1", "1.0"), new MetricsInfo("metric_2", "2.0"));
    records = Arrays.asList(new MetricsRecord("machine/stuff/record_1", infos, Collections.<ExceptionInfo>emptyList()), new MetricsRecord("record_2", infos, Collections.<ExceptionInfo>emptyList()));
}