Examples with MetricDataQuery co.cask.cdap.api.metrics.MetricDataQuery used on opensource projects

Example 21 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class Spark2Test method testSparkWithObjectStore.

@Test
public void testSparkWithObjectStore() throws Exception {
    ApplicationManager applicationManager = deploy(NamespaceId.DEFAULT, SparkAppUsingObjectStore.class);
    DataSetManager<ObjectStore<String>> keysManager = getDataset("keys");
    prepareInputData(keysManager);
    SparkManager sparkManager = applicationManager.getSparkManager(CharCountProgram.class.getSimpleName()).start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 1, TimeUnit.MINUTES);
    DataSetManager<KeyValueTable> countManager = getDataset("count");
    checkOutputData(countManager);
    // validate that the table emitted metrics
    // one read + one write in beforeSubmit(), increment (= read + write) in main -> 4
    Tasks.waitFor(4L, new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            Collection<MetricTimeSeries> metrics = getMetricsManager().query(new MetricDataQuery(0, System.currentTimeMillis() / 1000L, Integer.MAX_VALUE, "system." + Constants.Metrics.Name.Dataset.OP_COUNT, AggregationFunction.SUM, ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, DefaultId.NAMESPACE.getNamespace(), Constants.Metrics.Tag.APP, SparkAppUsingObjectStore.class.getSimpleName(), Constants.Metrics.Tag.SPARK, CharCountProgram.class.getSimpleName(), Constants.Metrics.Tag.DATASET, "totals"), Collections.<String>emptyList()));
            if (metrics.isEmpty()) {
                return 0L;
            }
            Assert.assertEquals(1, metrics.size());
            MetricTimeSeries ts = metrics.iterator().next();
            Assert.assertEquals(1, ts.getTimeValues().size());
            return ts.getTimeValues().get(0).getValue();
        }
    }, 10L, TimeUnit.SECONDS, 50L, TimeUnit.MILLISECONDS);
}

Example 22 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricsProcessorServiceTest method testMetricsProcessor.

@Test
public void testMetricsProcessor() throws Exception {
    injector.getInstance(TransactionManager.class).startAndWait();
    injector.getInstance(DatasetOpExecutor.class).startAndWait();
    injector.getInstance(DatasetService.class).startAndWait();
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    Set<Integer> partitions = new HashSet<>();
    for (int i = 0; i < PARTITION_SIZE; i++) {
        partitions.add(i);
    }
    // Start KafkaMetricsProcessorService after metrics are published to Kafka
    // Intentionally set queue size to a small value, so that MessagingMetricsProcessorService
    // internally can persist metrics when more messages are to be fetched
    MessagingMetricsProcessorService messagingMetricsProcessorService = new MessagingMetricsProcessorService(injector.getInstance(MetricDatasetFactory.class), TOPIC_PREFIX, messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, 1000L, 5, partitions, new NoopMetricsContext(), 50, 0, injector.getInstance(DatasetFramework.class), cConf, true);
    messagingMetricsProcessorService.startAndWait();
    long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // Publish metrics with messaging service and record expected metrics
    for (int i = 10; i < 20; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.COUNTER);
    }
    Thread.sleep(500);
    // Stop and restart messagingMetricsProcessorService
    messagingMetricsProcessorService.stopAndWait();
    // Intentionally set queue size to a large value, so that MessagingMetricsProcessorService
    // internally only persists metrics during terminating.
    messagingMetricsProcessorService = new MessagingMetricsProcessorService(injector.getInstance(MetricDatasetFactory.class), TOPIC_PREFIX, messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, 500L, 100, partitions, new NoopMetricsContext(), 50, 0, injector.getInstance(DatasetFramework.class), cConf, true);
    messagingMetricsProcessorService.startAndWait();
    // Publish metrics after MessagingMetricsProcessorService restarts and record expected metrics
    for (int i = 20; i < 30; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.GAUGE);
    }
    final List<String> missingMetricNames = new ArrayList<>();
    // are retrieved when timeout occurs, print out the missing metrics
    try {
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return canQueryAllMetrics(metricStore, METRICS_CONTEXT, expected, missingMetricNames);
            }
        }, 10000, TimeUnit.MILLISECONDS, "Failed to get all metrics");
    } catch (TimeoutException e) {
        Assert.fail(String.format("Metrics: [%s] cannot be found in the metrics store.", Joiner.on(", ").join(missingMetricNames)));
    }
    // Query metrics from the metricStore and compare them with the expected ones
    assertMetricsResult(metricStore, METRICS_CONTEXT, expected);
    // Query for the 5 counter metrics published with messaging between time 5 - 14
    Collection<MetricTimeSeries> queryResult = metricStore.query(new MetricDataQuery(5, 14, 1, Integer.MAX_VALUE, ImmutableMap.of(SYSTEM_METRIC_PREFIX + COUNTER_METRIC_NAME, AggregationFunction.SUM), METRICS_CONTEXT, ImmutableList.<String>of(), null));
    MetricTimeSeries timeSeries = Iterables.getOnlyElement(queryResult);
    Assert.assertEquals(5, timeSeries.getTimeValues().size());
    for (TimeValue timeValue : timeSeries.getTimeValues()) {
        Assert.assertEquals(1L, timeValue.getValue());
    }
    // Stop services and servers
    messagingMetricsProcessorService.stopAndWait();
    // Delete all metrics
    metricStore.deleteAll();
}

Example 23 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testCluster.

@Test
public void testCluster() throws MetricsPathException {
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/system/cluster/resources.total.storage?count=1&start=12345678&interpolate=step"));
    verifyTags(query.getSliceByTags(), "system");
    assertMetricName("system.resources.total.storage", query);
}

Example 24 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testMetricURIDecoding.

@Test
public void testMetricURIDecoding() throws UnsupportedEncodingException, MetricsPathException {
    String weirdMetric = "/weird?me+tr ic#$name////";
    // encoded version or weirdMetric
    String encodedWeirdMetric = "%2Fweird%3Fme%2Btr%20ic%23%24name%2F%2F%2F%2F";
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/user/apps/app1/flows/" + encodedWeirdMetric + "?aggregate=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1");
    assertMetricName("user." + weirdMetric, query);
    query = MetricQueryParser.parse(URI.create("/user/apps/app1/" + encodedWeirdMetric + "?aggregate=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId(), Tag.APP, "app1");
    assertMetricName("user." + weirdMetric, query);
}

Example 25 with MetricDataQuery

use of co.cask.cdap.api.metrics.MetricDataQuery in project cdap by caskdata.

the class MetricQueryParserTest method testStream.

@Test
public void testStream() throws MetricsPathException {
    MetricDataQuery query = MetricQueryParser.parse(URI.create("/system/streams/stream1/collect.events?summary=true"));
    verifyTags(query.getSliceByTags(), Id.Namespace.DEFAULT.getId());
    assertMetricName("system.collect.events", query);
    Assert.assertEquals("stream1", query.getSliceByTags().get(Tag.STREAM));
}