Examples with MetricDataQuery co.cask.cdap.api.metrics.MetricDataQuery used on opensource projects

Example 11 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testMapReduce.

@Test
public void testMapReduce() throws MetricsPathException {
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/system/apps/app1/mapreduce/mapred1/mappers/reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.MAPREDUCE, "mapred1", Tag.MR_TASK_TYPE, "m");
    assertMetricName("system.reads", query);
    query = MetricQueryParser.parse(URI.create("/system/apps/app1/mapreduce/mapred1/reducers/reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.MAPREDUCE, "mapred1", Tag.MR_TASK_TYPE, "r");
    assertMetricName("system.reads", query);
    query = MetricQueryParser.parse(URI.create("/system/apps/app1/mapreduce/mapred1/reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.MAPREDUCE, "mapred1");
    assertMetricName("system.reads", query);
    query = MetricQueryParser.parse(URI.create("/system/apps/app1/mapreduce/mapred1/runs/run123/reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.MAPREDUCE, "mapred1");
    assertMetricName("system.reads", query);
    Assert.assertEquals("run123", query.getSliceByTags().get(Tag.RUN_ID));
    query = MetricQueryParser.parse(URI.create("/system/apps/app1/mapreduce/mapred1/runs/run123/mappers/reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.MAPREDUCE, "mapred1", Tag.MR_TASK_TYPE, "m");
    assertMetricName("system.reads", query);
    Assert.assertEquals("run123", query.getSliceByTags().get(Tag.RUN_ID));
}

Example 12 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testHandler.

@Test
public void testHandler() throws MetricsPathException {
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/system/services/appfabric/handlers/AppFabricHttpHandler/runs/123/" + "response.server-error?aggregate=true"));
    verifyTags(query.getSliceByTags(), "system", Tag.COMPONENT, "appfabric", Tag.HANDLER, "AppFabricHttpHandler");
    assertMetricName("system.response.server-error", query);
    Assert.assertEquals("123", query.getSliceByTags().get(Tag.RUN_ID));
}

Example 13 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testDataset.

@Test
public void testDataset() throws MetricsPathException {
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/system/datasets/dataset1/apps/app1/flows/flow1/runs/run1/" + "flowlets/flowlet1/store.reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.FLOW, "flow1", Tag.FLOWLET, "flowlet1");
    assertMetricName("system.store.reads", query);
    Assert.assertEquals("dataset1", query.getSliceByTags().get(Tag.DATASET));
    Assert.assertEquals("run1", query.getSliceByTags().get(Tag.RUN_ID));
    query = MetricQueryParser.parse(URI.create("/system/datasets/dataset1/apps/app1/flows/flow1/store.reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.FLOW, "flow1");
    assertMetricName("system.store.reads", query);
    Assert.assertEquals("dataset1", query.getSliceByTags().get(Tag.DATASET));
    query = MetricQueryParser.parse(URI.create("/system/datasets/dataset1/apps/app1/flows/flow1/runs/123/store.reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1", Tag.FLOW, "flow1");
    assertMetricName("system.store.reads", query);
    Assert.assertEquals("dataset1", query.getSliceByTags().get(Tag.DATASET));
    Assert.assertEquals("123", query.getSliceByTags().get(Tag.RUN_ID));
    query = MetricQueryParser.parse(URI.create("/system/datasets/dataset1/apps/app1/store.reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1");
    assertMetricName("system.store.reads", query);
    Assert.assertEquals("dataset1", query.getSliceByTags().get(Tag.DATASET));
    query = MetricQueryParser.parse(URI.create("/system/datasets/dataset1/store.reads?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId());
    assertMetricName("system.store.reads", query);
    Assert.assertEquals("dataset1", query.getSliceByTags().get(Tag.DATASET));
}

Example 14 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricStoreRequestExecutor method computeFlowletPending.

private Object computeFlowletPending(MetricDataQuery query) throws Exception {
    // Pending is processed by flowlet minus emitted into the queues it was processing from.
    // trick: get all queues and streams it was processing from using group by
    MetricDataQuery groupByQueueName = new MetricDataQuery(new MetricDataQuery(query, "system.process.events.processed", AggregationFunction.SUM), ImmutableList.of(Constants.Metrics.Tag.FLOWLET_QUEUE));
    Map<String, Long> processedPerQueue = getTotalsWithSingleGroupByTag(groupByQueueName);
    long processedTotal = 0;
    long writtenTotal = 0;
    for (Map.Entry<String, Long> entry : processedPerQueue.entrySet()) {
        String name = entry.getKey();
        // note: each has "input." prefix
        QueueName queueName = QueueName.from(URI.create(name.substring("input.".length(), name.length())));
        long written;
        if (queueName.isQueue()) {
            Map<String, String> sliceByTags = Maps.newHashMap(query.getSliceByTags());
            // we want to aggregate written to the queue by all flowlets
            sliceByTags.remove(Constants.Metrics.Tag.FLOWLET);
            // we want to narrow down to specific queue we know our flowlet was consuming from
            sliceByTags.put(Constants.Metrics.Tag.FLOWLET_QUEUE, queueName.getSimpleName());
            written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags), "system.process.events.out", AggregationFunction.SUM));
        } else if (queueName.isStream()) {
            Map<String, String> sliceByTags = Maps.newHashMap();
            // we want to narrow down to specific stream we know our flowlet was consuming from
            sliceByTags.put(Constants.Metrics.Tag.STREAM, queueName.getSimpleName());
            // note: namespace + stream uniquely define the stream
            // we know that flow can consume from stream of the same namespace only at this point
            sliceByTags.put(Constants.Metrics.Tag.NAMESPACE, query.getSliceByTags().get(Constants.Metrics.Tag.NAMESPACE));
            written = getTotals(new MetricDataQuery(new MetricDataQuery(query, sliceByTags), "system.collect.events", AggregationFunction.SUM));
        } else {
            LOG.warn("Unknown queue type: " + name);
            continue;
        }
        processedTotal += entry.getValue();
        writtenTotal += written;
    }
    long pending = writtenTotal - processedTotal;
    return new AggregateResponse(pending > 0 ? pending : 0);
}

Example 15 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class TestFrameworkTestRun method verifyMapperJobOutput.

private void verifyMapperJobOutput(Class<?> appClass, DataSetManager<KeyValueTable> outTableManager) throws Exception {
    KeyValueTable outputTable = outTableManager.get();
    Assert.assertEquals("world", Bytes.toString(outputTable.read("hello")));
    // Verify dataset metrics
    String readCountName = "system." + Constants.Metrics.Name.Dataset.READ_COUNT;
    String writeCountName = "system." + Constants.Metrics.Name.Dataset.WRITE_COUNT;
    Collection<MetricTimeSeries> metrics = getMetricsManager().query(new MetricDataQuery(0, System.currentTimeMillis() / 1000, Integer.MAX_VALUE, ImmutableMap.of(readCountName, AggregationFunction.SUM, writeCountName, AggregationFunction.SUM), ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, DefaultId.NAMESPACE.getNamespace(), Constants.Metrics.Tag.APP, appClass.getSimpleName(), Constants.Metrics.Tag.MAPREDUCE, DatasetWithMRApp.MAPREDUCE_PROGRAM), ImmutableList.<String>of()));
    // Transform the collection of metrics into a map from metrics name to aggregated sum
    Map<String, Long> aggs = Maps.transformEntries(Maps.uniqueIndex(metrics, new Function<MetricTimeSeries, String>() {

        @Override
        public String apply(MetricTimeSeries input) {
            return input.getMetricName();
        }
    }), new Maps.EntryTransformer<String, MetricTimeSeries, Long>() {

        @Override
        public Long transformEntry(String key, MetricTimeSeries value) {
            Preconditions.checkArgument(value.getTimeValues().size() == 1, "Expected one value for aggregated sum for metrics %s", key);
            return value.getTimeValues().get(0).getValue();
        }
    });
    Assert.assertEquals(Long.valueOf(1), aggs.get(readCountName));
    Assert.assertEquals(Long.valueOf(1), aggs.get(writeCountName));
}