Examples with ExceptionInfo org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo used on opensource projects

Example 1 with ExceptionInfo

use of org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class MetricsCacheSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    LOG.info("metricscache sink processRecord");
    // Format it into TopologyManager.PublishMetrics
    // The format of record is "host:port/componentName/instanceId"
    // So MetricsRecord.getSource().split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = MetricsUtil.splitRecordSource(record);
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyManager.PublishMetrics.Builder publishMetrics = TopologyManager.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tManagerMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyManager.MetricDatum metricDatum = TopologyManager.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        String exceptionStackTrace = exceptionInfo.getStackTrace();
        String[] exceptionStackTraceLines = exceptionStackTrace.split("\r\n|[\r\n]", 3);
        String exceptionStackTraceFirstTwoLines = String.join(System.lineSeparator(), exceptionStackTraceLines[0], exceptionStackTraceLines[1]);
        TopologyManager.TmanagerExceptionLog exceptionLog = TopologyManager.TmanagerExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionStackTraceFirstTwoLines).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
    checkCommunicator(metricsCommunicator, MAX_COMMUNICATOR_SIZE);
}

Example 2 with ExceptionInfo

use of org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class WebSinkTest method testGroupedMetrics.

/**
 * Testing grouped map with metrics
 */
@Test
public void testGroupedMetrics() {
    Map<String, Object> conf = new HashMap<>(defaultConf);
    conf.put("flat-metrics", "false");
    WebTestSink sink = new WebTestSink();
    sink.init(conf, context);
    for (MetricsRecord r : records) {
        sink.processRecord(r);
    }
    // Update and override MetricsRecord 1
    Iterable<MetricsInfo> infos2 = Arrays.asList(new MetricsInfo("metric_1", "3.0"), new MetricsInfo("metric_3", "1.0"));
    sink.processRecord(new MetricsRecord(records.get(0).getSource(), infos2, Collections.<ExceptionInfo>emptyList()));
    Map<String, Object> results = sink.getMetrics();
    Assert.assertEquals(2, results.size());
    @SuppressWarnings("unchecked") Map<String, Object> record1 = (Map<String, Object>) results.get("/stuff/record_1");
    @SuppressWarnings("unchecked") Map<String, Object> record2 = (Map<String, Object>) results.get("/record_2");
    Assert.assertEquals(record1.get("metric_1"), 3.0d);
    Assert.assertEquals(record1.get("metric_2"), 2.0d);
    Assert.assertEquals(record1.get("metric_3"), 1.0d);
    Assert.assertEquals(record2.get("metric_1"), 1.0d);
    Assert.assertEquals(record2.get("metric_2"), 2.0d);
}

Example 3 with ExceptionInfo

use of org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class MetricsManagerServer method handlePublisherPublishMessage.

private void handlePublisherPublishMessage(Metrics.MetricPublisher request, Metrics.MetricPublisherPublishMessage message) {
    if (message.getMetricsCount() <= 0 && message.getExceptionsCount() <= 0) {
        LOG.log(Level.SEVERE, "Publish message has no metrics nor exceptions for message from hostname: {0}," + " component_name: {1}, port: {2}, instance_id: {3}, instance_index: {4}", new Object[] { request.getHostname(), request.getComponentName(), request.getPort(), request.getInstanceId(), request.getInstanceIndex() });
        return;
    }
    // Convert the message to MetricsRecord
    String source = MetricsUtil.createSource(request.getHostname(), request.getPort(), request.getComponentName(), request.getInstanceId());
    List<MetricsInfo> metricsInfos = new ArrayList<MetricsInfo>(message.getMetricsCount());
    for (Metrics.MetricDatum metricDatum : message.getMetricsList()) {
        MetricsInfo info = new MetricsInfo(metricDatum.getName(), metricDatum.getValue());
        metricsInfos.add(info);
    }
    List<ExceptionInfo> exceptionInfos = new ArrayList<ExceptionInfo>(message.getExceptionsCount());
    for (Metrics.ExceptionData exceptionData : message.getExceptionsList()) {
        ExceptionInfo exceptionInfo = new ExceptionInfo(exceptionData.getStacktrace(), exceptionData.getLasttime(), exceptionData.getFirsttime(), exceptionData.getCount(), exceptionData.getLogging());
        exceptionInfos.add(exceptionInfo);
    }
    LOG.info(String.format("%d MetricsInfo and %d ExceptionInfo to push", metricsInfos.size(), exceptionInfos.size()));
    // Update the metrics
    serverMetricsCounters.scope(SERVER_METRICS_RECEIVED).incrBy(metricsInfos.size());
    serverMetricsCounters.scope(SERVER_EXCEPTIONS_RECEIVED).incrBy(exceptionInfos.size());
    MetricsRecord record = new MetricsRecord(source, metricsInfos, exceptionInfos);
    // Push MetricsRecord to Communicator, which would wake up ExecutorLooper bind with IMetricsSink
    synchronized (metricsSinkCommunicators) {
        Iterator<String> itr = metricsSinkCommunicators.keySet().iterator();
        while (itr.hasNext()) {
            String key = itr.next();
            Communicator<MetricsRecord> c = metricsSinkCommunicators.get(key);
            c.offer(record);
            serverMetricsCounters.scope(SERVER_COMMUNICATOR_OFFER).incr();
            serverMetricsCounters.scope(SERVER_COMMUNICATOR_SIZE + "-" + key).incrBy(c.size());
        }
    }
}

Example 4 with ExceptionInfo

use of org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class TManagerSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    // Format it into TopologyManager.PublishMetrics
    // The format of record is "host:port/componentName/instanceId"
    // So MetricsRecord.getSource().split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = MetricsUtil.splitRecordSource(record);
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyManager.PublishMetrics.Builder publishMetrics = TopologyManager.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tManagerMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyManager.MetricDatum metricDatum = TopologyManager.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyManager.TmanagerExceptionLog exceptionLog = TopologyManager.TmanagerExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
    checkCommunicator(metricsCommunicator, MAX_COMMUNICATOR_SIZE);
}

Example 5 with ExceptionInfo

use of org.apache.heron.spi.metricsmgr.metrics.ExceptionInfo in project heron by twitter.

the class FileSink method convertRecordToJSON.

private String convertRecordToJSON(MetricsRecord record) {
    int metricsCount = 0;
    int exceptionsCount = 0;
    // Pack metrics as a map
    Map<String, String> metrics = new HashMap<String, String>();
    for (MetricsInfo metricsInfo : record.getMetrics()) {
        metrics.put(metricsInfo.getName(), metricsInfo.getValue());
        metricsCount++;
    }
    // Pack exceptions as a list of map
    LinkedList<Object> exceptions = new LinkedList<Object>();
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        Map<String, Object> exception = new HashMap<String, Object>();
        exception.put("firstTime", exceptionInfo.getFirstTime());
        exception.put("lastTime", exceptionInfo.getLastTime());
        exception.put("logging", exceptionInfo.getLogging());
        exception.put("stackTrace", exceptionInfo.getStackTrace());
        exception.put("count", exceptionInfo.getCount());
        exceptions.add(exception);
        exceptionsCount++;
    }
    // Pack the whole MetricsRecord as a map
    Map<String, Object> jsonToWrite = new HashMap<String, Object>();
    jsonToWrite.put("timestamp", record.getTimestamp());
    jsonToWrite.put("source", record.getSource());
    jsonToWrite.put("context", record.getContext());
    jsonToWrite.put("metrics", metrics);
    jsonToWrite.put("exceptions", exceptions);
    // Update metrics
    sinkContext.exportCountMetric(METRICS_COUNT, metricsCount);
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, exceptionsCount);
    String result = "";
    try {
        result = MAPPER.writeValueAsString(jsonToWrite);
    } catch (JsonProcessingException e) {
        LOG.log(Level.SEVERE, "Could not convert map to JSONString: " + jsonToWrite.toString(), e);
    }
    return result;
}