Examples with MetricsContext co.cask.cdap.api.metrics.MetricsContext used on opensource projects

Example 6 with MetricsContext

use of co.cask.cdap.api.metrics.MetricsContext in project cdap by caskdata.

the class MetricsLogAppender method append.

@Override
protected void append(ILoggingEvent eventObject) {
    // increment metrics for logging event
    Map<String, String> metricsTags = getMetricsTagsMap(eventObject.getMDCPropertyMap());
    if (!metricsTags.isEmpty()) {
        String metricName = getMetricName(metricsTags.get(Constants.Metrics.Tag.NAMESPACE), eventObject.getLevel().toString().toLowerCase());
        // Don't increment metrics for logs from MetricsProcessor to avoid possibility of infinite loop
        if (!(metricsTags.containsKey(Constants.Metrics.Tag.COMPONENT) && metricsTags.get(Constants.Metrics.Tag.COMPONENT).equals(Constants.Service.METRICS_PROCESSOR))) {
            // todo this is inefficient as childContext implementation creates new map should use metricsCollectionService
            MetricsContext childContext = metricsContext.childContext(metricsTags);
            childContext.increment(metricName, 1);
        }
    }
}

Example 7 with MetricsContext

use of co.cask.cdap.api.metrics.MetricsContext in project cdap by caskdata.

the class AggregatedMetricsCollectionServiceTest method testPublish.

@Category(SlowTests.class)
@Test
public void testPublish() throws InterruptedException {
    final BlockingQueue<MetricValues> published = new LinkedBlockingQueue<>();
    AggregatedMetricsCollectionService service = new AggregatedMetricsCollectionService() {

        @Override
        protected void publish(Iterator<MetricValues> metrics) {
            Iterators.addAll(published, metrics);
        }

        @Override
        protected long getInitialDelayMillis() {
            return 5000L;
        }

        @Override
        protected long getPeriodMillis() {
            return 1000L;
        }
    };
    service.startAndWait();
    // non-empty tags.
    final Map baseTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NAMESPACE, Constants.Metrics.Tag.APP, APP, Constants.Metrics.Tag.FLOW, FLOW, Constants.Metrics.Tag.RUN_ID, RUNID);
    try {
        // The first section tests with empty tags.
        // Publish couple metrics with empty tags, they should be aggregated.
        service.getContext(EMPTY_TAGS).increment(METRIC, Integer.MAX_VALUE);
        service.getContext(EMPTY_TAGS).increment(METRIC, 2);
        service.getContext(EMPTY_TAGS).increment(METRIC, 3);
        service.getContext(EMPTY_TAGS).increment(METRIC, 4);
        MetricValues record = published.poll(10, TimeUnit.SECONDS);
        Assert.assertNotNull(record);
        Assert.assertEquals(((long) Integer.MAX_VALUE) + 9L, getMetricValue(record.getMetrics(), METRIC));
        // No publishing for 0 value metrics
        Assert.assertNull(published.poll(3, TimeUnit.SECONDS));
        // Publish a metric and wait for it so that we know there is around 1 second to publish more metrics to test.
        service.getContext(EMPTY_TAGS).increment(METRIC, 1);
        Assert.assertNotNull(published.poll(3, TimeUnit.SECONDS));
        //update the metrics multiple times with gauge.
        service.getContext(EMPTY_TAGS).gauge(GAUGE_METRIC, 1);
        service.getContext(EMPTY_TAGS).gauge(GAUGE_METRIC, 2);
        service.getContext(EMPTY_TAGS).gauge(GAUGE_METRIC, 3);
        // gauge just updates the value, so polling should return the most recent value written
        record = published.poll(3, TimeUnit.SECONDS);
        Assert.assertNotNull(record);
        Assert.assertEquals(3, getMetricValue(record.getMetrics(), GAUGE_METRIC));
        // define collectors for non-empty tags
        MetricsContext baseCollector = service.getContext(baseTags);
        MetricsContext flowletInstanceCollector = baseCollector.childContext(Constants.Metrics.Tag.FLOWLET, FLOWLET).childContext(Constants.Metrics.Tag.INSTANCE_ID, INSTANCE);
        // increment metrics for various collectors
        baseCollector.increment(METRIC, Integer.MAX_VALUE);
        flowletInstanceCollector.increment(METRIC, 5);
        baseCollector.increment(METRIC, 10);
        baseCollector.increment(METRIC, 3);
        flowletInstanceCollector.increment(METRIC, 2);
        flowletInstanceCollector.increment(METRIC, 4);
        flowletInstanceCollector.increment(METRIC, 3);
        flowletInstanceCollector.increment(METRIC, 1);
        // there are two collectors, verify their metrics values
        verifyCounterMetricsValue(published.poll(10, TimeUnit.SECONDS));
        verifyCounterMetricsValue(published.poll(10, TimeUnit.SECONDS));
        // No publishing for 0 value metrics
        Assert.assertNull(published.poll(3, TimeUnit.SECONDS));
        // gauge metrics for various collectors
        baseCollector.gauge(GAUGE_METRIC, Integer.MAX_VALUE);
        baseCollector.gauge(GAUGE_METRIC, 3);
        flowletInstanceCollector.gauge(GAUGE_METRIC, 6);
        flowletInstanceCollector.gauge(GAUGE_METRIC, 2);
        baseCollector.gauge(GAUGE_METRIC, 1);
        flowletInstanceCollector.gauge(GAUGE_METRIC, Integer.MAX_VALUE);
        // gauge just updates the value, so polling should return the most recent value written
        verifyGaugeMetricsValue(published.poll(10, TimeUnit.SECONDS));
        verifyGaugeMetricsValue(published.poll(10, TimeUnit.SECONDS));
        flowletInstanceCollector.gauge(GAUGE_METRIC, 0);
        verifyMetricsValue(published.poll(10, TimeUnit.SECONDS), 0L, GAUGE_METRIC);
    } finally {
        service.stopAndWait();
    }
}

Example 8 with MetricsContext

use of co.cask.cdap.api.metrics.MetricsContext in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

/**
   * Corrects queue.pending metric for a flowlet.
   */
public void run(FlowId flowId, String producerFlowlet, String consumerFlowlet, String flowletQueue, FlowSpecification flow) throws Exception {
    System.out.println("Running queue.pending correction on flow '" + flowId + "' producerFlowlet '" + producerFlowlet + "' consumerFlowlet '" + consumerFlowlet + "' flowletQueue '" + flowletQueue + "'");
    Map<RunId, ProgramRuntimeService.RuntimeInfo> runtimeInfos = programRuntimeService.list(flowId);
    Preconditions.checkState(runtimeInfos.isEmpty(), "Cannot run tool when flow " + flowId + " is still running");
    SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
    Preconditions.checkArgument(table.contains(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet), "Flowlet " + producerFlowlet + " is not emitting to " + consumerFlowlet);
    Set<QueueSpecification> queueSpecs = table.get(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet);
    boolean validQueue = false;
    for (QueueSpecification queueSpec : queueSpecs) {
        if (queueSpec.getQueueName().getSimpleName().equals(flowletQueue)) {
            validQueue = true;
            break;
        }
    }
    Preconditions.checkArgument(validQueue, "Queue " + flowletQueue + " does not exist for the given flowlets");
    QueueName queueName = QueueName.fromFlowlet(flowId, producerFlowlet, flowletQueue);
    long consumerGroupId = FlowUtils.generateConsumerGroupId(flowId, consumerFlowlet);
    long correctQueuePendingValue;
    try {
        HBaseQueueDebugger.QueueStatistics stats = queueDebugger.scanQueue(queueName, consumerGroupId);
        correctQueuePendingValue = stats.getUnprocessed() + stats.getProcessedAndNotVisible();
    } catch (NotFoundException e) {
        // OK since flowlet queue exists, but actual queue doesn't exist
        // (e.g. when running upgrade tool from 2.8 to 3.0)
        correctQueuePendingValue = 0;
    }
    Map<String, String> tags = ImmutableMap.<String, String>builder().put(Constants.Metrics.Tag.NAMESPACE, flowId.getNamespace()).put(Constants.Metrics.Tag.APP, flowId.getApplication()).put(Constants.Metrics.Tag.FLOW, flowId.getProgram()).put(Constants.Metrics.Tag.CONSUMER, consumerFlowlet).put(Constants.Metrics.Tag.PRODUCER, producerFlowlet).put(Constants.Metrics.Tag.FLOWLET_QUEUE, flowletQueue).build();
    MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, 1, ImmutableMap.of("system.queue.pending", AggregationFunction.SUM), tags, ImmutableList.<String>of(), null);
    Collection<MetricTimeSeries> results = metricStore.query(query);
    long queuePending;
    if (results.isEmpty()) {
        queuePending = 0;
    } else {
        System.out.println("Got results: " + GSON.toJson(results));
        Preconditions.checkState(results.size() == 1);
        List<TimeValue> timeValues = results.iterator().next().getTimeValues();
        Preconditions.checkState(timeValues.size() == 1);
        TimeValue timeValue = timeValues.get(0);
        queuePending = timeValue.getValue();
    }
    metricsCollectionService.startAndWait();
    MetricsContext collector = metricsCollectionService.getContext(tags);
    collector.gauge("queue.pending", correctQueuePendingValue);
    System.out.printf("Adjusted system.queue.pending metric from %d to %d (tags %s)\n", queuePending, correctQueuePendingValue, GSON.toJson(tags));
    // stop will flush the metrics
    metricsCollectionService.stopAndWait();
}

Example 9 with MetricsContext

use of co.cask.cdap.api.metrics.MetricsContext in project cdap by caskdata.

the class AbstractResourceReporter method sendMetrics.

protected void sendMetrics(Map<String, String> context, int containers, int memory, int vcores) {
    LOG.trace("Reporting resources: (containers, memory, vcores) = ({}, {}, {})", containers, memory, vcores);
    MetricsContext metricsContext = programMetricsCollectors.getUnchecked(context);
    metricsContext.gauge(METRIC_CONTAINERS, containers);
    metricsContext.gauge(METRIC_MEMORY_USAGE, memory);
    metricsContext.gauge(METRIC_VIRTUAL_CORE_USAGE, vcores);
}

Example 10 with MetricsContext

use of co.cask.cdap.api.metrics.MetricsContext in project cdap by caskdata.

the class MetricsReporterHook method postCall.

@Override
public void postCall(HttpRequest request, HttpResponseStatus status, HandlerInfo handlerInfo) {
    if (metricsCollectionService != null) {
        try {
            MetricsContext collector = collectorCache.get(createContext(handlerInfo));
            String name;
            int code = status.getCode();
            if (code < 100) {
                name = "unknown";
            } else if (code < 200) {
                name = "information";
            } else if (code < 300) {
                name = "successful";
            } else if (code < 400) {
                name = "redirect";
            } else if (code < 500) {
                name = "client-error";
            } else if (code < 600) {
                name = "server-error";
            } else {
                name = "unknown";
            }
            // todo: report metrics broken down by status
            collector.increment("response." + name, 1);
        } catch (Throwable e) {
            LOG.error("Got exception while getting collector", e);
        }
    }
}