Examples with ExceptionInfo com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo used on opensource projects

Example 1 with ExceptionInfo

use of com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class WebSinkTest method testGroupedMetrics.

/**
   * Testing grouped map with metrics
   */
@Test
public void testGroupedMetrics() {
    Map<String, Object> conf = new HashMap<>(defaultConf);
    conf.put("flat-metrics", "false");
    WebTestSink sink = new WebTestSink();
    sink.init(conf, context);
    for (MetricsRecord r : records) {
        sink.processRecord(r);
    }
    //Update and override MetricsRecord 1
    Iterable<MetricsInfo> infos2 = Arrays.asList(new MetricsInfo("metric_1", "3.0"), new MetricsInfo("metric_3", "1.0"));
    sink.processRecord(new MetricsRecord(records.get(0).getSource(), infos2, Collections.<ExceptionInfo>emptyList()));
    Map<String, Object> results = sink.getMetrics();
    Assert.assertEquals(2, results.size());
    @SuppressWarnings("unchecked") Map<String, Object> record1 = (Map<String, Object>) results.get("/stuff/record_1");
    @SuppressWarnings("unchecked") Map<String, Object> record2 = (Map<String, Object>) results.get("/record_2");
    Assert.assertEquals(record1.get("metric_1"), 3.0d);
    Assert.assertEquals(record1.get("metric_2"), 2.0d);
    Assert.assertEquals(record1.get("metric_3"), 1.0d);
    Assert.assertEquals(record2.get("metric_1"), 1.0d);
    Assert.assertEquals(record2.get("metric_2"), 2.0d);
}

Example 2 with ExceptionInfo

use of com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class TMasterSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    // Format it into TopologyMaster.PublishMetrics
    // The format of source is "host:port/componentName/instanceId"
    // So source.split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = record.getSource().split("/");
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyMaster.PublishMetrics.Builder publishMetrics = TopologyMaster.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tMasterMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyMaster.MetricDatum metricDatum = TopologyMaster.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyMaster.TmasterExceptionLog exceptionLog = TopologyMaster.TmasterExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
}

Example 3 with ExceptionInfo

use of com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class MetricsManagerServerTest method testMetricsManagerServer.

/**
   * Method: addSinkCommunicator(Communicator<MetricsRecord> communicator)
   */
@Test
public void testMetricsManagerServer() throws Exception {
    final Communicator<MetricsRecord> sinkCommunicator = new Communicator<MetricsRecord>();
    metricsManagerServer.addSinkCommunicator(sinkCommunicator);
    // First run Server
    runServer();
    // Wait a while for server fully starting
    Thread.sleep(3 * 1000);
    // Then run Client
    runClient();
    // Wait some while to let message fully send out
    Thread.sleep(10 * 1000);
    int messages = 0;
    while (!sinkCommunicator.isEmpty()) {
        int exceptions = 0;
        int metrics = 0;
        MetricsRecord record = sinkCommunicator.poll();
        Assert.assertEquals("hostname:0/component/instance-id", record.getSource());
        Assert.assertEquals("default", record.getContext());
        for (MetricsInfo info : record.getMetrics()) {
            Assert.assertEquals(METRIC_NAME, info.getName());
            Assert.assertEquals(METRIC_VALUE, info.getValue());
            metrics++;
        }
        Assert.assertEquals(N, metrics);
        for (ExceptionInfo info : record.getExceptions()) {
            Assert.assertEquals(STACK_TRACE, info.getStackTrace());
            Assert.assertEquals(LAST_TIME, info.getLastTime());
            Assert.assertEquals(FIRST_TIME, info.getFirstTime());
            Assert.assertEquals(LOGGING, info.getLogging());
            Assert.assertEquals(EXCEPTION_COUNT, info.getCount());
            exceptions++;
        }
        Assert.assertEquals(N, exceptions);
        messages++;
    }
    Assert.assertEquals(MESSAGE_SIZE, messages);
}

Example 4 with ExceptionInfo

use of com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class FileSink method convertRecordToJSON.

private String convertRecordToJSON(MetricsRecord record) {
    int metricsCount = 0;
    int exceptionsCount = 0;
    // Pack metrics as a map
    Map<String, String> metrics = new HashMap<String, String>();
    for (MetricsInfo metricsInfo : record.getMetrics()) {
        metrics.put(metricsInfo.getName(), metricsInfo.getValue());
        metricsCount++;
    }
    // Pack exceptions as a list of map
    LinkedList<Object> exceptions = new LinkedList<Object>();
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        Map<String, Object> exception = new HashMap<String, Object>();
        exception.put("firstTime", exceptionInfo.getFirstTime());
        exception.put("lastTime", exceptionInfo.getLastTime());
        exception.put("logging", exceptionInfo.getLogging());
        exception.put("stackTrace", exceptionInfo.getStackTrace());
        exception.put("count", exceptionInfo.getCount());
        exceptions.add(exception);
        exceptionsCount++;
    }
    // Pack the whole MetricsRecord as a map
    Map<String, Object> jsonToWrite = new HashMap<String, Object>();
    jsonToWrite.put("timestamp", record.getTimestamp());
    jsonToWrite.put("source", record.getSource());
    jsonToWrite.put("context", record.getContext());
    jsonToWrite.put("metrics", metrics);
    jsonToWrite.put("exceptions", exceptions);
    // Update metrics
    sinkContext.exportCountMetric(METRICS_COUNT, metricsCount);
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, exceptionsCount);
    String result = "";
    try {
        result = MAPPER.writeValueAsString(jsonToWrite);
    } catch (JsonProcessingException e) {
        LOG.log(Level.SEVERE, "Could not convert map to JSONString: " + jsonToWrite.toString(), e);
    }
    return result;
}

Example 5 with ExceptionInfo

use of com.twitter.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class MetricsCacheSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    LOG.info("metricscache sink processRecord");
    // Format it into TopologyMaster.PublishMetrics
    // The format of source is "host:port/componentName/instanceId"
    // So source.split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = record.getSource().split("/");
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyMaster.PublishMetrics.Builder publishMetrics = TopologyMaster.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tMasterMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyMaster.MetricDatum metricDatum = TopologyMaster.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyMaster.TmasterExceptionLog exceptionLog = TopologyMaster.TmasterExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
}