Examples with MetricTimeSeries co.cask.cdap.api.metrics.MetricTimeSeries used on opensource projects

Example 6 with MetricTimeSeries

use of co.cask.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class FlowQueuePendingCorrector method run.

/**
   * Corrects queue.pending metric for a flowlet.
   */
public void run(FlowId flowId, String producerFlowlet, String consumerFlowlet, String flowletQueue, FlowSpecification flow) throws Exception {
    System.out.println("Running queue.pending correction on flow '" + flowId + "' producerFlowlet '" + producerFlowlet + "' consumerFlowlet '" + consumerFlowlet + "' flowletQueue '" + flowletQueue + "'");
    Map<RunId, ProgramRuntimeService.RuntimeInfo> runtimeInfos = programRuntimeService.list(flowId);
    Preconditions.checkState(runtimeInfos.isEmpty(), "Cannot run tool when flow " + flowId + " is still running");
    SimpleQueueSpecificationGenerator queueSpecGenerator = new SimpleQueueSpecificationGenerator(flowId.getParent());
    Table<QueueSpecificationGenerator.Node, String, Set<QueueSpecification>> table = queueSpecGenerator.create(flow);
    Preconditions.checkArgument(table.contains(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet), "Flowlet " + producerFlowlet + " is not emitting to " + consumerFlowlet);
    Set<QueueSpecification> queueSpecs = table.get(QueueSpecificationGenerator.Node.flowlet(producerFlowlet), consumerFlowlet);
    boolean validQueue = false;
    for (QueueSpecification queueSpec : queueSpecs) {
        if (queueSpec.getQueueName().getSimpleName().equals(flowletQueue)) {
            validQueue = true;
            break;
        }
    }
    Preconditions.checkArgument(validQueue, "Queue " + flowletQueue + " does not exist for the given flowlets");
    QueueName queueName = QueueName.fromFlowlet(flowId, producerFlowlet, flowletQueue);
    long consumerGroupId = FlowUtils.generateConsumerGroupId(flowId, consumerFlowlet);
    long correctQueuePendingValue;
    try {
        HBaseQueueDebugger.QueueStatistics stats = queueDebugger.scanQueue(queueName, consumerGroupId);
        correctQueuePendingValue = stats.getUnprocessed() + stats.getProcessedAndNotVisible();
    } catch (NotFoundException e) {
        // OK since flowlet queue exists, but actual queue doesn't exist
        // (e.g. when running upgrade tool from 2.8 to 3.0)
        correctQueuePendingValue = 0;
    }
    Map<String, String> tags = ImmutableMap.<String, String>builder().put(Constants.Metrics.Tag.NAMESPACE, flowId.getNamespace()).put(Constants.Metrics.Tag.APP, flowId.getApplication()).put(Constants.Metrics.Tag.FLOW, flowId.getProgram()).put(Constants.Metrics.Tag.CONSUMER, consumerFlowlet).put(Constants.Metrics.Tag.PRODUCER, producerFlowlet).put(Constants.Metrics.Tag.FLOWLET_QUEUE, flowletQueue).build();
    MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, 1, ImmutableMap.of("system.queue.pending", AggregationFunction.SUM), tags, ImmutableList.<String>of(), null);
    Collection<MetricTimeSeries> results = metricStore.query(query);
    long queuePending;
    if (results.isEmpty()) {
        queuePending = 0;
    } else {
        System.out.println("Got results: " + GSON.toJson(results));
        Preconditions.checkState(results.size() == 1);
        List<TimeValue> timeValues = results.iterator().next().getTimeValues();
        Preconditions.checkState(timeValues.size() == 1);
        TimeValue timeValue = timeValues.get(0);
        queuePending = timeValue.getValue();
    }
    metricsCollectionService.startAndWait();
    MetricsContext collector = metricsCollectionService.getContext(tags);
    collector.gauge("queue.pending", correctQueuePendingValue);
    System.out.printf("Adjusted system.queue.pending metric from %d to %d (tags %s)\n", queuePending, correctQueuePendingValue, GSON.toJson(tags));
    // stop will flush the metrics
    metricsCollectionService.stopAndWait();
}

Example 7 with MetricTimeSeries

use of co.cask.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class TestFrameworkTestRun method verifyMapperJobOutput.

private void verifyMapperJobOutput(Class<?> appClass, DataSetManager<KeyValueTable> outTableManager) throws Exception {
    KeyValueTable outputTable = outTableManager.get();
    Assert.assertEquals("world", Bytes.toString(outputTable.read("hello")));
    // Verify dataset metrics
    String readCountName = "system." + Constants.Metrics.Name.Dataset.READ_COUNT;
    String writeCountName = "system." + Constants.Metrics.Name.Dataset.WRITE_COUNT;
    Collection<MetricTimeSeries> metrics = getMetricsManager().query(new MetricDataQuery(0, System.currentTimeMillis() / 1000, Integer.MAX_VALUE, ImmutableMap.of(readCountName, AggregationFunction.SUM, writeCountName, AggregationFunction.SUM), ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, DefaultId.NAMESPACE.getNamespace(), Constants.Metrics.Tag.APP, appClass.getSimpleName(), Constants.Metrics.Tag.MAPREDUCE, DatasetWithMRApp.MAPREDUCE_PROGRAM), ImmutableList.<String>of()));
    // Transform the collection of metrics into a map from metrics name to aggregated sum
    Map<String, Long> aggs = Maps.transformEntries(Maps.uniqueIndex(metrics, new Function<MetricTimeSeries, String>() {

        @Override
        public String apply(MetricTimeSeries input) {
            return input.getMetricName();
        }
    }), new Maps.EntryTransformer<String, MetricTimeSeries, Long>() {

        @Override
        public Long transformEntry(String key, MetricTimeSeries value) {
            Preconditions.checkArgument(value.getTimeValues().size() == 1, "Expected one value for aggregated sum for metrics %s", key);
            return value.getTimeValues().get(0).getValue();
        }
    });
    Assert.assertEquals(Long.valueOf(1), aggs.get(readCountName));
    Assert.assertEquals(Long.valueOf(1), aggs.get(writeCountName));
}

Example 8 with MetricTimeSeries

use of co.cask.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class MetricStoreRequestExecutor method getTotalsWithSingleGroupByTag.

private Map<String, Long> getTotalsWithSingleGroupByTag(MetricDataQuery query) throws Exception {
    // query must have resolution set to Integer.MAX_VALUE (i.e. "totals")
    Collection<MetricTimeSeries> result = metricStore.query(query);
    Map<String, Long> map = Maps.newHashMap();
    for (MetricTimeSeries timeSeries : result) {
        // we know there's only ony group by tag
        String groupByTagValue = timeSeries.getTagValues().values().iterator().next();
        // since it is totals, it will have only one TimeValue
        map.put(groupByTagValue, timeSeries.getTimeValues().get(0).getValue());
    }
    return map;
}

Example 9 with MetricTimeSeries

use of co.cask.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class MetricStoreRequestExecutor method getTotals.

private long getTotals(MetricDataQuery query) throws Exception {
    // query must have resolution set to Integer.MAX_VALUE (i.e. "totals")
    Collection<MetricTimeSeries> result = metricStore.query(query);
    if (result.size() == 0) {
        return 0;
    }
    // since there's no group by condition, it'll return single time series always
    MetricTimeSeries timeSeries = result.iterator().next();
    if (timeSeries.getTimeValues().isEmpty()) {
        return 0;
    }
    // since it is totals, it will have only one TimeValue or none
    return timeSeries.getTimeValues().get(0).getValue();
}

Example 10 with MetricTimeSeries

use of co.cask.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class CDAPLoad method collect.

@Override
public void collect() throws IOException {
    // reset all metrics
    reset();
    long currentTimeSecs = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // We want metrics in the last hour
    long startTimeSecs = currentTimeSecs - (60 * 60);
    // We want to aggregate metrics in a sliding window of the past hour from the time the request is made.
    // To do this, query metrics with the minute resolution, so you get 60 points, then aggregate on the client side.
    // Not using hourly resolution here, because that wouldn't give aggregated metrics for the past hour, but
    // just aggregated metrics for the current hour (if called 15 mins past the hour, it will only give an aggregated
    // value for the past 15 mins).
    Collection<MetricTimeSeries> metricTimeSeries = metricStore.query(// TODO: CDAP-8207 Have to use this constructor currently, because the default limit does not work
    new MetricDataQuery(startTimeSecs, currentTimeSecs, 60, Integer.MAX_VALUE, METRICS, Collections.singletonMap(Constants.Metrics.Tag.NAMESPACE, NamespaceId.SYSTEM.getEntityName()), Collections.<String>emptyList(), null));
    for (MetricTimeSeries metricTimeSery : metricTimeSeries) {
        switch(metricTimeSery.getMetricName()) {
            case "system.request.received":
                totalRequests = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.successful":
                successful = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.client-error":
                clientErrors = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.server-error":
                serverErrors = aggregateMetricValue(metricTimeSery);
                break;
            case "system.services.log.error":
                errorLogs = aggregateMetricValue(metricTimeSery);
                break;
            case "system.services.log.warn":
                warnLogs = aggregateMetricValue(metricTimeSery);
                break;
        }
    }
}