Examples with MetricTimeSeries io.cdap.cdap.api.metrics.MetricTimeSeries used on opensource projects

Example 11 with MetricTimeSeries

use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class SparkMetricsIntegrationTestRun method getTotalCounter.

private long getTotalCounter(Map<String, String> context, String metricName) throws Exception {
    MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>());
    try {
        Collection<MetricTimeSeries> result = getMetricsManager().query(query);
        if (result.isEmpty()) {
            return 0;
        }
        // since it is totals query and not groupBy specified, we know there's one time series
        List<TimeValue> timeValues = result.iterator().next().getTimeValues();
        if (timeValues.isEmpty()) {
            return 0;
        }
        // since it is totals, we know there's one value only
        return timeValues.get(0).getValue();
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 12 with MetricTimeSeries

use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class MetricsProcessorServiceTest method testMetricsProcessor.

@Test
public void testMetricsProcessor() throws Exception {
    injector.getInstance(TransactionManager.class).startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    injector.getInstance(DatasetOpExecutorService.class).startAndWait();
    injector.getInstance(DatasetService.class).startAndWait();
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    Set<Integer> partitions = new HashSet<>();
    for (int i = 0; i < cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM); i++) {
        partitions.add(i);
    }
    // Start KafkaMetricsProcessorService after metrics are published to Kafka
    // Intentionally set queue size to a small value, so that MessagingMetricsProcessorManagerService
    // internally can persist metrics when more messages are to be fetched
    MessagingMetricsProcessorManagerService messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorManagerService.startAndWait();
    long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // Publish metrics with messaging service and record expected metrics
    for (int i = 10; i < 20; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.COUNTER);
    }
    Thread.sleep(500);
    // Stop and restart messagingMetricsProcessorManagerService
    messagingMetricsProcessorManagerService.stopAndWait();
    // Intentionally set queue size to a large value, so that MessagingMetricsProcessorManagerService
    // internally only persists metrics during terminating.
    messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorManagerService.startAndWait();
    // Publish metrics after MessagingMetricsProcessorManagerService restarts and record expected metrics
    for (int i = 20; i < 30; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.GAUGE);
    }
    final List<String> missingMetricNames = new ArrayList<>();
    // are retrieved when timeout occurs, print out the missing metrics
    try {
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return canQueryAllMetrics(metricStore, METRICS_CONTEXT, expected, missingMetricNames);
            }
        }, 10000, TimeUnit.MILLISECONDS, "Failed to get all metrics");
    } catch (TimeoutException e) {
        Assert.fail(String.format("Metrics: [%s] cannot be found in the metrics store.", Joiner.on(", ").join(missingMetricNames)));
    }
    // Query metrics from the metricStore and compare them with the expected ones
    assertMetricsResult(metricStore, METRICS_CONTEXT, expected);
    // Query for the 5 counter metrics published with messaging between time 5 - 14
    Collection<MetricTimeSeries> queryResult = metricStore.query(new MetricDataQuery(5, 14, 1, Integer.MAX_VALUE, ImmutableMap.of(SYSTEM_METRIC_PREFIX + COUNTER_METRIC_NAME, AggregationFunction.SUM), METRICS_CONTEXT, ImmutableList.<String>of(), null));
    MetricTimeSeries timeSeries = Iterables.getOnlyElement(queryResult);
    Assert.assertEquals(5, timeSeries.getTimeValues().size());
    for (TimeValue timeValue : timeSeries.getTimeValues()) {
        Assert.assertEquals(1L, timeValue.getValue());
    }
    // Stop services and servers
    messagingMetricsProcessorManagerService.stopAndWait();
    // Delete all metrics
    metricStore.deleteAll();
}

Example 13 with MetricTimeSeries

use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class CDAPLoad method collect.

@Override
public void collect() throws IOException {
    // reset all metrics
    reset();
    int currentTimeSecs = (int) TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // We want metrics in the last hour
    int startTimeSecs = currentTimeSecs - (60 * 60);
    // We want to aggregate metrics in a sliding window of the past hour from the time the request is made.
    // To do this, query metrics with the minute resolution, so you get 60 points, then aggregate on the client side.
    // Not using hourly resolution here, because that wouldn't give aggregated metrics for the past hour, but
    // just aggregated metrics for the current hour (if called 15 mins past the hour, it will only give an aggregated
    // value for the past 15 mins).
    Map<String, String> tags = Collections.singletonMap(Constants.Metrics.Tag.NAMESPACE, NamespaceId.SYSTEM.getEntityName());
    Collection<MetricTimeSeries> metricTimeSeries = metricsSystemClient.query(startTimeSecs, currentTimeSecs, 60, tags, METRICS, Collections.emptySet());
    for (MetricTimeSeries metricTimeSery : metricTimeSeries) {
        switch(metricTimeSery.getMetricName()) {
            case "system.request.received":
                totalRequests = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.successful":
                successful = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.client-error":
                clientErrors = aggregateMetricValue(metricTimeSery);
                break;
            case "system.response.server-error":
                serverErrors = aggregateMetricValue(metricTimeSery);
                break;
            case "system.services.log.error":
                errorLogs = aggregateMetricValue(metricTimeSery);
                break;
            case "system.services.log.warn":
                warnLogs = aggregateMetricValue(metricTimeSery);
                break;
        }
    }
}

Example 14 with MetricTimeSeries

use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by caskdata.

the class CDAPTransactions method collect.

@Override
public void collect() throws Exception {
    Collection<MetricTimeSeries> collection = metricsSystemClient.query(Constants.Metrics.TRANSACTION_MANAGER_CONTEXT, METRICS);
    for (MetricTimeSeries metricTimeSeries : collection) {
        if (metricTimeSeries.getMetricName().equals("system.committing.size")) {
            numCommittingChangeSets = (int) aggregateMetricValue(metricTimeSeries);
        }
        if (metricTimeSeries.getMetricName().equals("system.committed.size")) {
            numCommittedChangeSets = (int) aggregateMetricValue(metricTimeSeries);
        }
    }
    Transaction transaction = txClient.startShort();
    readPointer = transaction.getReadPointer();
    writePointer = transaction.getWritePointer();
    numInProgressTx = transaction.getInProgress().length;
    numInvalidTx = transaction.getInvalids().length;
    txClient.abort(transaction);
}

Example 15 with MetricTimeSeries

use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.

the class ProfileMetricServiceTest method getMetric.

private long getMetric(MetricStore metricStore, ProgramRunId programRunId, ProfileId profileId, String metricName) {
    Map<String, String> tags = ImmutableMap.<String, String>builder().put(Constants.Metrics.Tag.PROFILE_SCOPE, profileId.getScope().name()).put(Constants.Metrics.Tag.PROFILE, profileId.getProfile()).put(Constants.Metrics.Tag.NAMESPACE, programRunId.getNamespace()).put(Constants.Metrics.Tag.PROGRAM_TYPE, programRunId.getType().getPrettyName()).put(Constants.Metrics.Tag.APP, programRunId.getApplication()).put(Constants.Metrics.Tag.PROGRAM, programRunId.getProgram()).build();
    MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, tags, new ArrayList<>());
    Collection<MetricTimeSeries> result = metricStore.query(query);
    if (result.isEmpty()) {
        return 0;
    }
    List<TimeValue> timeValues = result.iterator().next().getTimeValues();
    if (timeValues.isEmpty()) {
        return 0;
    }
    return timeValues.get(0).getValue();
}