Examples with MetricsRecord org.apache.hadoop.metrics2.MetricsRecord used on opensource projects

Example 1 with MetricsRecord

use of org.apache.hadoop.metrics2.MetricsRecord in project hadoop by apache.

the class TestKafkaMetrics method recordToJson.

StringBuilder recordToJson(MetricsRecord record) {
    // Create a json object from a metrics record.
    StringBuilder jsonLines = new StringBuilder();
    Long timestamp = record.timestamp();
    Date currDate = new Date(timestamp);
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
    String date = dateFormat.format(currDate);
    SimpleDateFormat timeFormat = new SimpleDateFormat("hh:mm:ss");
    String time = timeFormat.format(currDate);
    String hostname = new String("null");
    try {
        hostname = InetAddress.getLocalHost().getHostName();
    } catch (Exception e) {
        LOG.warn("Error getting Hostname, going to continue");
    }
    jsonLines.append("{\"hostname\": \"" + hostname);
    jsonLines.append("\", \"timestamp\": " + timestamp);
    jsonLines.append(", \"date\": \"" + date);
    jsonLines.append("\",\"time\": \"" + time);
    jsonLines.append("\",\"name\": \"" + record.name() + "\" ");
    for (MetricsTag tag : record.tags()) {
        jsonLines.append(", \"" + tag.name().toString().replaceAll("[\\p{Cc}]", "") + "\": ");
        jsonLines.append(" \"" + tag.value().toString() + "\"");
    }
    for (AbstractMetric m : record.metrics()) {
        jsonLines.append(", \"" + m.name().toString().replaceAll("[\\p{Cc}]", "") + "\": ");
        jsonLines.append(" \"" + m.value().toString() + "\"");
    }
    jsonLines.append("}");
    return jsonLines;
}

Example 2 with MetricsRecord

use of org.apache.hadoop.metrics2.MetricsRecord in project hadoop by apache.

the class TestKafkaMetrics method testPutMetrics.

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testPutMetrics() throws Exception {
    // Create a record by mocking MetricsRecord class.
    MetricsRecord record = mock(MetricsRecord.class);
    when(record.tags()).thenReturn(Lists.newArrayList(new MetricsTag(KafkaMetricsInfo.KafkaTag, "test_tag")));
    when(record.timestamp()).thenReturn(System.currentTimeMillis());
    // Create a metric using AbstractMetric class.
    AbstractMetric metric = new AbstractMetric(KafkaMetricsInfo.KafkaCounter) {

        @Override
        public Number value() {
            return new Integer(123);
        }

        @Override
        public MetricType type() {
            return null;
        }

        @Override
        public void visit(MetricsVisitor visitor) {
        }
    };
    // Create a list of metrics.
    Iterable<AbstractMetric> metrics = Lists.newArrayList(metric);
    when(record.name()).thenReturn("Kafka record name");
    when(record.metrics()).thenReturn(metrics);
    SubsetConfiguration conf = mock(SubsetConfiguration.class);
    when(conf.getString(KafkaSink.BROKER_LIST)).thenReturn("localhost:9092");
    String topic = "myTestKafkaTopic";
    when(conf.getString(KafkaSink.TOPIC)).thenReturn(topic);
    // Create the KafkaSink object and initialize it.
    kafkaSink = new KafkaSink();
    kafkaSink.init(conf);
    // Create a mock KafkaProducer as a producer for KafkaSink.
    Producer<Integer, byte[]> mockProducer = mock(KafkaProducer.class);
    kafkaSink.setProducer(mockProducer);
    // Create the json object from the record.
    StringBuilder jsonLines = recordToJson(record);
    if (LOG.isDebugEnabled()) {
        LOG.debug("kafka message: " + jsonLines.toString());
    }
    // Send the record and store the result in a mock Future.
    Future<RecordMetadata> f = mock(Future.class);
    when(mockProducer.send((ProducerRecord) anyObject())).thenReturn(f);
    kafkaSink.putMetrics(record);
    // Get the argument and verity it.
    ArgumentCaptor<ProducerRecord> argument = ArgumentCaptor.forClass(ProducerRecord.class);
    verify(mockProducer).send(argument.capture());
    // Compare the received data with the original one.
    ProducerRecord<Integer, byte[]> data = (argument.getValue());
    String jsonResult = new String(data.value());
    if (LOG.isDebugEnabled()) {
        LOG.debug("kafka result: " + jsonResult);
    }
    assertEquals(jsonLines.toString(), jsonResult);
}

Example 3 with MetricsRecord

use of org.apache.hadoop.metrics2.MetricsRecord in project hadoop by apache.

the class MetricsCache method update.

/**
   * Update the cache and return the current cached record
   * @param mr the update record
   * @param includingTags cache tag values (for later lookup by name) if true
   * @return the updated cache record
   */
public Record update(MetricsRecord mr, boolean includingTags) {
    String name = mr.name();
    RecordCache recordCache = map.get(name);
    if (recordCache == null) {
        recordCache = new RecordCache();
        map.put(name, recordCache);
    }
    Collection<MetricsTag> tags = mr.tags();
    Record record = recordCache.get(tags);
    if (record == null) {
        record = new Record();
        recordCache.put(tags, record);
    }
    for (AbstractMetric m : mr.metrics()) {
        record.metrics.put(m.name(), m);
    }
    if (includingTags) {
        // mostly for some sinks that include tags as part of a dense schema
        for (MetricsTag t : mr.tags()) {
            record.tags.put(t.name(), t.value());
        }
    }
    return record;
}

Example 4 with MetricsRecord

use of org.apache.hadoop.metrics2.MetricsRecord in project hadoop by apache.

the class RollingFileSystemSink method putMetrics.

@Override
public void putMetrics(MetricsRecord record) {
    synchronized (lock) {
        rollLogDirIfNeeded();
        if (currentOutStream != null) {
            currentOutStream.printf("%d %s.%s", record.timestamp(), record.context(), record.name());
            String separator = ": ";
            for (MetricsTag tag : record.tags()) {
                currentOutStream.printf("%s%s=%s", separator, tag.name(), tag.value());
                separator = ", ";
            }
            for (AbstractMetric metric : record.metrics()) {
                currentOutStream.printf("%s%s=%s", separator, metric.name(), metric.value());
            }
            currentOutStream.println();
            // is complete at the end of the interval.
            try {
                currentFSOutStream.hflush();
            } catch (IOException ex) {
                throwMetricsException("Failed flushing the stream", ex);
            }
            checkForErrors("Unable to write to log file");
        } else if (!ignoreError) {
            throwMetricsException("Unable to write to log file");
        }
    }
}

Example 5 with MetricsRecord

use of org.apache.hadoop.metrics2.MetricsRecord in project hadoop by apache.

the class GraphiteSink method putMetrics.

@Override
public void putMetrics(MetricsRecord record) {
    StringBuilder lines = new StringBuilder();
    StringBuilder metricsPathPrefix = new StringBuilder();
    // Configure the hierarchical place to display the graph.
    metricsPathPrefix.append(metricsPrefix).append(".").append(record.context()).append(".").append(record.name());
    for (MetricsTag tag : record.tags()) {
        if (tag.value() != null) {
            metricsPathPrefix.append(".");
            metricsPathPrefix.append(tag.name());
            metricsPathPrefix.append("=");
            metricsPathPrefix.append(tag.value());
        }
    }
    // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds.
    long timestamp = record.timestamp() / 1000L;
    // Collect datapoints.
    for (AbstractMetric metric : record.metrics()) {
        lines.append(metricsPathPrefix.toString() + "." + metric.name().replace(' ', '.')).append(" ").append(metric.value()).append(" ").append(timestamp).append("\n");
    }
    try {
        graphite.write(lines.toString());
    } catch (Exception e) {
        LOG.warn("Error sending metrics to Graphite", e);
        try {
            graphite.close();
        } catch (Exception e1) {
            throw new MetricsException("Error closing connection to Graphite", e1);
        }
    }
}