Examples with MetricsInfo com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo used on opensource projects

Example 6 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class FileSink method convertRecordToJSON.

private String convertRecordToJSON(MetricsRecord record) {
    int metricsCount = 0;
    int exceptionsCount = 0;
    // Pack metrics as a map
    Map<String, String> metrics = new HashMap<String, String>();
    for (MetricsInfo metricsInfo : record.getMetrics()) {
        metrics.put(metricsInfo.getName(), metricsInfo.getValue());
        metricsCount++;
    }
    // Pack exceptions as a list of map
    LinkedList<Object> exceptions = new LinkedList<Object>();
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        Map<String, Object> exception = new HashMap<String, Object>();
        exception.put("firstTime", exceptionInfo.getFirstTime());
        exception.put("lastTime", exceptionInfo.getLastTime());
        exception.put("logging", exceptionInfo.getLogging());
        exception.put("stackTrace", exceptionInfo.getStackTrace());
        exception.put("count", exceptionInfo.getCount());
        exceptions.add(exception);
        exceptionsCount++;
    }
    // Pack the whole MetricsRecord as a map
    Map<String, Object> jsonToWrite = new HashMap<String, Object>();
    jsonToWrite.put("timestamp", record.getTimestamp());
    jsonToWrite.put("source", record.getSource());
    jsonToWrite.put("context", record.getContext());
    jsonToWrite.put("metrics", metrics);
    jsonToWrite.put("exceptions", exceptions);
    // Update metrics
    sinkContext.exportCountMetric(METRICS_COUNT, metricsCount);
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, exceptionsCount);
    String result = "";
    try {
        result = MAPPER.writeValueAsString(jsonToWrite);
    } catch (JsonProcessingException e) {
        LOG.log(Level.SEVERE, "Could not convert map to JSONString: " + jsonToWrite.toString(), e);
    }
    return result;
}

Example 7 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class ScribeSink method makeJSON.

// Convert a record into json format required by Twitter Cuckoo
private String makeJSON(MetricsRecord record) {
    String serviceName = String.format("%s/%s", config.get(KEY_SERVICE_NAMESPACE), topologyName);
    // The source format is "host:port/componentName/instanceId"
    // However, we need just "/componentName/instanceId"
    String[] sources = record.getSource().split("/");
    String source = String.format("/%s/%s", sources[1], sources[2]);
    // The timestamp is in ms, however, we need to convert it in seconds to fit Twitter Infra
    long timestamp = record.getTimestamp() / Constants.SECONDS_TO_MILLISECONDS;
    Map<String, Object> json = new HashMap<String, Object>();
    // Add the service name
    json.put("service", serviceName);
    // Add the service name
    json.put("source", source);
    // Add the time stamp
    json.put("timestamp", timestamp);
    // Cuckoo_json allows multi-metrics in a single JSON, so we would like to
    // package all metrics received into one single JSON
    int metricsToWrite = 0;
    for (MetricsInfo info : record.getMetrics()) {
        // First check whether the metric  value is legal
        // since scribe would only accept a Double value as metric value
        // We would just skip it
        Double val;
        try {
            val = Double.valueOf(info.getValue());
        } catch (NumberFormatException ne) {
            LOG.log(Level.SEVERE, "Could not parse illegal metric: " + info.toString());
            sinkContext.exportCountMetric(ILLEGAL_METRICS_COUNT, 1);
            continue;
        }
        json.put(info.getName(), val);
        metricsToWrite++;
    }
    LOG.info(metricsToWrite + " metrics added");
    String result = "";
    try {
        result = MAPPER.writeValueAsString(json);
    } catch (JsonProcessingException e) {
        LOG.log(Level.SEVERE, "Could not convert map to JSONString: " + json.toString(), e);
    }
    return result;
}

Example 8 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class MetricsCacheSink method processRecord.

@Override
public void processRecord(MetricsRecord record) {
    LOG.info("metricscache sink processRecord");
    // Format it into TopologyMaster.PublishMetrics
    // The format of source is "host:port/componentName/instanceId"
    // So source.split("/") would be an array with 3 elements:
    // ["host:port", componentName, instanceId]
    String[] sources = record.getSource().split("/");
    String hostPort = sources[0];
    String componentName = sources[1];
    String instanceId = sources[2];
    TopologyMaster.PublishMetrics.Builder publishMetrics = TopologyMaster.PublishMetrics.newBuilder();
    for (MetricsInfo metricsInfo : tMasterMetricsFilter.filter(record.getMetrics())) {
        // We would filter out unneeded metrics
        TopologyMaster.MetricDatum metricDatum = TopologyMaster.MetricDatum.newBuilder().setComponentName(componentName).setInstanceId(instanceId).setName(metricsInfo.getName()).setValue(metricsInfo.getValue()).setTimestamp(record.getTimestamp()).build();
        publishMetrics.addMetrics(metricDatum);
    }
    for (ExceptionInfo exceptionInfo : record.getExceptions()) {
        TopologyMaster.TmasterExceptionLog exceptionLog = TopologyMaster.TmasterExceptionLog.newBuilder().setComponentName(componentName).setHostname(hostPort).setInstanceId(instanceId).setStacktrace(exceptionInfo.getStackTrace()).setLasttime(exceptionInfo.getLastTime()).setFirsttime(exceptionInfo.getFirstTime()).setCount(exceptionInfo.getCount()).setLogging(exceptionInfo.getLogging()).build();
        publishMetrics.addExceptions(exceptionLog);
    }
    metricsCommunicator.offer(publishMetrics.build());
    // Update metrics
    sinkContext.exportCountMetric(RECORD_PROCESS_COUNT, 1);
    sinkContext.exportCountMetric(METRICS_COUNT, publishMetrics.getMetricsCount());
    sinkContext.exportCountMetric(EXCEPTIONS_COUNT, publishMetrics.getExceptionsCount());
}

Example 9 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class SinkExecutorTest method constructMetricsRecord.

private MetricsRecord constructMetricsRecord() {
    List<MetricsInfo> metricsInfos = new ArrayList<>();
    // We add N MetricsInfo into a MetricsRecord
    for (int i = 0; i < N; i++) {
        MetricsInfo metricsInfo = new MetricsInfo(METRICS_NAME + i, METRICS_VALUE + i);
        metricsInfos.add(metricsInfo);
    }
    // We add N ExceptionInfo into a MetricsRecord
    List<ExceptionInfo> exceptionInfos = new ArrayList<>();
    for (int i = 0; i < N; i++) {
        String stackTrace = EXCEPTION_STACK_TRACE + i;
        String lastTime = EXCEPTION_LAST_TIME + i;
        String firstTime = EXCEPTION_FIRST_TIME + i;
        String logging = EXCEPTION_LOGGING + i;
        ExceptionInfo info = new ExceptionInfo(stackTrace, lastTime, firstTime, i, logging);
        exceptionInfos.add(info);
    }
    return new MetricsRecord(RECORD_SOURCE, metricsInfos, exceptionInfos, RECORD_CONTEXT);
}

Example 10 with MetricsInfo

use of com.twitter.heron.spi.metricsmgr.metrics.MetricsInfo in project heron by twitter.

the class MetricsManagerServer method handlePublisherPublishMessage.

private void handlePublisherPublishMessage(Metrics.MetricPublisher request, Metrics.MetricPublisherPublishMessage message) {
    if (message.getMetricsCount() <= 0 && message.getExceptionsCount() <= 0) {
        LOG.log(Level.SEVERE, "Publish message has no metrics nor exceptions for message from hostname: {0}," + " component_name: {1}, port: {2}, instance_id: {3}, instance_index: {4}", new Object[] { request.getHostname(), request.getComponentName(), request.getPort(), request.getInstanceId(), request.getInstanceIndex() });
        return;
    }
    // Convert the message to MetricsRecord
    String source = String.format("%s:%d/%s/%s", request.getHostname(), request.getPort(), request.getComponentName(), request.getInstanceId());
    List<MetricsInfo> metricsInfos = new ArrayList<MetricsInfo>(message.getMetricsCount());
    for (Metrics.MetricDatum metricDatum : message.getMetricsList()) {
        MetricsInfo info = new MetricsInfo(metricDatum.getName(), metricDatum.getValue());
        metricsInfos.add(info);
    }
    List<ExceptionInfo> exceptionInfos = new ArrayList<ExceptionInfo>(message.getExceptionsCount());
    for (Metrics.ExceptionData exceptionData : message.getExceptionsList()) {
        ExceptionInfo exceptionInfo = new ExceptionInfo(exceptionData.getStacktrace(), exceptionData.getLasttime(), exceptionData.getFirsttime(), exceptionData.getCount(), exceptionData.getLogging());
        exceptionInfos.add(exceptionInfo);
    }
    LOG.info(String.format("%d MetricsInfo and %d ExceptionInfo to push", metricsInfos.size(), exceptionInfos.size()));
    // Update the metrics
    serverMetricsCounters.scope(SERVER_METRICS_RECEIVED).incrBy(metricsInfos.size());
    serverMetricsCounters.scope(SERVER_EXCEPTIONS_RECEIVED).incrBy(exceptionInfos.size());
    MetricsRecord record = new MetricsRecord(source, metricsInfos, exceptionInfos);
    // Push MetricsRecord to Communicator, which would wake up SlaveLooper bind with IMetricsSink
    for (Communicator<MetricsRecord> c : metricsSinkCommunicators) {
        c.offer(record);
    }
}