use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.
the class ServiceLifeCycleTestRun method testContentConsumerLifecycle.
@Test
public void testContentConsumerLifecycle() throws Exception {
try {
ApplicationManager appManager = deployWithArtifact(ServiceLifecycleApp.class, artifactJar);
// Set to have one thread only for testing context capture and release
serviceManager = appManager.getServiceManager("test").start(ImmutableMap.of(SystemArguments.SERVICE_THREADS, "1"));
CountDownLatch uploadLatch = new CountDownLatch(1);
// Create five concurrent upload
List<ListenableFuture<Integer>> completions = new ArrayList<>();
for (int i = 0; i < 5; i++) {
completions.add(slowUpload(serviceManager, "PUT", "upload", uploadLatch));
}
// Get the states, there should be six handler instances initialized.
// Five for the in-progress upload, one for the getStates call
Tasks.waitFor(6, () -> getStates(serviceManager).size(), 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Finish the upload
uploadLatch.countDown();
Futures.successfulAsList(completions).get(10, TimeUnit.SECONDS);
// Verify the result
for (ListenableFuture<Integer> future : completions) {
Assert.assertEquals(200, future.get().intValue());
}
// Get the states, there should still be six handler instances initialized.
final Multimap<Integer, String> states = getStates(serviceManager);
Assert.assertEquals(6, states.size());
// Do another round of six concurrent upload. It should reuse all of the existing six contexts
completions.clear();
uploadLatch = new CountDownLatch(1);
for (int i = 0; i < 6; i++) {
completions.add(slowUpload(serviceManager, "PUT", "upload", uploadLatch));
}
// Get the states, there should be seven handler instances initialized.
// Six for the in-progress upload, one for the getStates call
// Out of the 7 states, six of them should be the same as the old one
Tasks.waitFor(true, () -> {
Multimap<Integer, String> newStates = getStates(serviceManager);
if (newStates.size() != 7) {
return false;
}
for (Map.Entry<Integer, String> entry : states.entries()) {
if (!newStates.containsEntry(entry.getKey(), entry.getValue())) {
return false;
}
}
return true;
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Complete the upload
uploadLatch.countDown();
Futures.successfulAsList(completions).get(10, TimeUnit.SECONDS);
// Verify the result
for (ListenableFuture<Integer> future : completions) {
Assert.assertEquals(200, future.get().intValue());
}
// Query the queue size metrics. Expect the maximum be 6.
// This is because only the six from the concurrent upload will get captured added back to the queue,
// while the one created for the getState() call will be stated in the thread cache, but not in the queue.
Tasks.waitFor(6L, () -> {
Map<String, String> context = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NamespaceId.DEFAULT.getNamespace(), Constants.Metrics.Tag.APP, ServiceLifecycleApp.class.getSimpleName(), Constants.Metrics.Tag.SERVICE, "test");
MetricDataQuery metricQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.context.pool.size", AggregationFunction.MAX, context, Collections.emptyList());
Iterator<MetricTimeSeries> result = getMetricsManager().query(metricQuery).iterator();
return result.hasNext() ? result.next().getTimeValues().get(0).getValue() : 0L;
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
} finally {
serviceManager.stop();
serviceManager.waitForStopped(10, TimeUnit.SECONDS);
}
}
use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.
the class SparkMetricsIntegrationTestRun method getTotalCounter.
private long getTotalCounter(Map<String, String> context, String metricName) throws Exception {
MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>());
try {
Collection<MetricTimeSeries> result = getMetricsManager().query(query);
if (result.isEmpty()) {
return 0;
}
// since it is totals query and not groupBy specified, we know there's one time series
List<TimeValue> timeValues = result.iterator().next().getTimeValues();
if (timeValues.isEmpty()) {
return 0;
}
// since it is totals, we know there's one value only
return timeValues.get(0).getValue();
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.
the class MetricsProcessorServiceTest method testMetricsProcessor.
@Test
public void testMetricsProcessor() throws Exception {
injector.getInstance(TransactionManager.class).startAndWait();
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
injector.getInstance(DatasetOpExecutorService.class).startAndWait();
injector.getInstance(DatasetService.class).startAndWait();
final MetricStore metricStore = injector.getInstance(MetricStore.class);
Set<Integer> partitions = new HashSet<>();
for (int i = 0; i < cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM); i++) {
partitions.add(i);
}
// Start KafkaMetricsProcessorService after metrics are published to Kafka
// Intentionally set queue size to a small value, so that MessagingMetricsProcessorManagerService
// internally can persist metrics when more messages are to be fetched
MessagingMetricsProcessorManagerService messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
messagingMetricsProcessorManagerService.startAndWait();
long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
// Publish metrics with messaging service and record expected metrics
for (int i = 10; i < 20; i++) {
publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.COUNTER);
}
Thread.sleep(500);
// Stop and restart messagingMetricsProcessorManagerService
messagingMetricsProcessorManagerService.stopAndWait();
// Intentionally set queue size to a large value, so that MessagingMetricsProcessorManagerService
// internally only persists metrics during terminating.
messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
messagingMetricsProcessorManagerService.startAndWait();
// Publish metrics after MessagingMetricsProcessorManagerService restarts and record expected metrics
for (int i = 20; i < 30; i++) {
publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.GAUGE);
}
final List<String> missingMetricNames = new ArrayList<>();
// are retrieved when timeout occurs, print out the missing metrics
try {
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return canQueryAllMetrics(metricStore, METRICS_CONTEXT, expected, missingMetricNames);
}
}, 10000, TimeUnit.MILLISECONDS, "Failed to get all metrics");
} catch (TimeoutException e) {
Assert.fail(String.format("Metrics: [%s] cannot be found in the metrics store.", Joiner.on(", ").join(missingMetricNames)));
}
// Query metrics from the metricStore and compare them with the expected ones
assertMetricsResult(metricStore, METRICS_CONTEXT, expected);
// Query for the 5 counter metrics published with messaging between time 5 - 14
Collection<MetricTimeSeries> queryResult = metricStore.query(new MetricDataQuery(5, 14, 1, Integer.MAX_VALUE, ImmutableMap.of(SYSTEM_METRIC_PREFIX + COUNTER_METRIC_NAME, AggregationFunction.SUM), METRICS_CONTEXT, ImmutableList.<String>of(), null));
MetricTimeSeries timeSeries = Iterables.getOnlyElement(queryResult);
Assert.assertEquals(5, timeSeries.getTimeValues().size());
for (TimeValue timeValue : timeSeries.getTimeValues()) {
Assert.assertEquals(1L, timeValue.getValue());
}
// Stop services and servers
messagingMetricsProcessorManagerService.stopAndWait();
// Delete all metrics
metricStore.deleteAll();
}
use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.
the class Spark2Test method testSparkWithObjectStore.
@Test
public void testSparkWithObjectStore() throws Exception {
ApplicationManager applicationManager = deploy(NamespaceId.DEFAULT, SparkAppUsingObjectStore.class);
DataSetManager<ObjectStore<String>> keysManager = getDataset("keys");
prepareInputData(keysManager);
SparkManager sparkManager = applicationManager.getSparkManager(CharCountProgram.class.getSimpleName()).start();
sparkManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
sparkManager.waitForStopped(60, TimeUnit.SECONDS);
DataSetManager<KeyValueTable> countManager = getDataset("count");
checkOutputData(countManager);
// validate that the table emitted metrics
// one read + one write in beforeSubmit(), increment (= read + write) in main -> 4
Tasks.waitFor(4L, new Callable<Long>() {
@Override
public Long call() throws Exception {
Collection<MetricTimeSeries> metrics = getMetricsManager().query(new MetricDataQuery(0, System.currentTimeMillis() / 1000L, Integer.MAX_VALUE, "system." + Constants.Metrics.Name.Dataset.OP_COUNT, AggregationFunction.SUM, ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, DefaultId.NAMESPACE.getNamespace(), Constants.Metrics.Tag.APP, SparkAppUsingObjectStore.class.getSimpleName(), Constants.Metrics.Tag.SPARK, CharCountProgram.class.getSimpleName(), Constants.Metrics.Tag.DATASET, "totals"), Collections.<String>emptyList()));
if (metrics.isEmpty()) {
return 0L;
}
Assert.assertEquals(1, metrics.size());
MetricTimeSeries ts = metrics.iterator().next();
Assert.assertEquals(1, ts.getTimeValues().size());
return ts.getTimeValues().get(0).getValue();
}
}, 10L, TimeUnit.SECONDS, 50L, TimeUnit.MILLISECONDS);
}
use of io.cdap.cdap.api.metrics.MetricTimeSeries in project cdap by cdapio.
the class DefaultMetricStore method query.
@Override
public Collection<MetricTimeSeries> query(MetricDataQuery query) {
Collection<TimeSeries> cubeResult = cube.get().query(buildCubeQuery(query));
List<MetricTimeSeries> result = Lists.newArrayList();
for (TimeSeries timeSeries : cubeResult) {
result.add(new MetricTimeSeries(timeSeries.getMeasureName(), timeSeries.getDimensionValues(), timeSeries.getTimeValues()));
}
return result;
}
Aggregations