Examples with MetricStore io.cdap.cdap.api.metrics.MetricStore used on opensource projects

Example 6 with MetricStore

use of io.cdap.cdap.api.metrics.MetricStore in project cdap by cdapio.

the class ProfileMetricServiceTest method testRoundingLogic.

@Test
public void testRoundingLogic() throws Exception {
    ProgramRunId runId = NamespaceId.DEFAULT.app("round").workflow("round").run(RunIds.generate());
    ProfileId profileId = NamespaceId.DEFAULT.profile("roundProfile");
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    MetricStore metricStore = injector.getInstance(MetricStore.class);
    ProfileMetricService scheduledService = new ProfileMetricService(collectionService, runId, profileId, 1, 1);
    // start and stop the service, the metric should still go up by 1
    scheduledService.startUp();
    scheduledService.shutDown();
    Tasks.waitFor(1L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.startUp();
    // set the start up time to 90 seconds before the current time
    scheduledService.setStartUpTime(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) - 90);
    // 90 seconds should round up to 2 mins, so emit 1 min and test the rounding logic
    scheduledService.emitMetric();
    scheduledService.shutDown();
    // the metric should go up by 2
    Tasks.waitFor(3L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.startUp();
    // set the start up time to 65 seconds before the current time
    scheduledService.setStartUpTime(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) - 65);
    // 65 seconds should round down to 1 min, so emit 1 min and test the rest seconds are ignored
    scheduledService.emitMetric();
    scheduledService.shutDown();
    // the metric should go up by 1
    Tasks.waitFor(4L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
}

Example 7 with MetricStore

use of io.cdap.cdap.api.metrics.MetricStore in project cdap by cdapio.

the class ProfileServiceTest method testProfileMetricsDeletion.

@Test
public void testProfileMetricsDeletion() throws Exception {
    ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
    Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate());
    // create and disable the profile
    profileService.saveProfile(myProfile, profile);
    profileService.disableProfile(myProfile);
    // emit some metrics
    MetricsCollectionService metricService = getInjector().getInstance(MetricsCollectionService.class);
    MetricsContext metricsContext = metricService.getContext(getMetricsTags(runId, myProfile));
    metricsContext.increment(Constants.Metrics.Program.PROGRAM_NODE_MINUTES, 30L);
    MetricStore metricStore = getInjector().getInstance(MetricStore.class);
    Tasks.waitFor(30L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    // delete and verify the metrics are gone
    profileService.deleteProfile(myProfile);
    Tasks.waitFor(0L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
}

Example 8 with MetricStore

use of io.cdap.cdap.api.metrics.MetricStore in project cdap by cdapio.

the class KafkaLogProcessorPipelineTest method testMetricsAppender.

@Test
public void testMetricsAppender() throws Exception {
    Injector injector = KAFKA_TESTER.getInjector();
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    collectionService.startAndWait();
    LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class));
    final File logDir = TEMP_FOLDER.newFolder();
    loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
    LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
    configurator.setContext(loggerContext);
    URL configURL = getClass().getClassLoader().getResource("pipeline-metric-appender.xml");
    Assert.assertNotNull(configURL);
    configurator.doConfigure(configURL);
    String topic = "metricsPipeline";
    TestCheckpointManager checkpointManager = new TestCheckpointManager();
    KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
    KAFKA_TESTER.createTopic(topic, 1);
    loggerContext.start();
    KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMetricAppender", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
    pipeline.startAndWait();
    // Publish some log messages to Kafka
    long now = System.currentTimeMillis();
    WorkerLoggingContext loggingContext = new WorkerLoggingContext("default", "app1", "worker1", "run1", "instance1");
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "3", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 900), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.DEBUG, "hidden", now - 600), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "4", now - 100)), loggingContext);
    WorkflowProgramLoggingContext workflowProgramLoggingContext = new WorkflowProgramLoggingContext("default", "app1", "wflow1", "run1", ProgramType.MAPREDUCE, "mr1", "mrun1");
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.WARN, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.WARN, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.TRACE, "3", now - 500)), workflowProgramLoggingContext);
    ServiceLoggingContext serviceLoggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION);
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "3", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 900)), serviceLoggingContext);
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    try {
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.info", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(loggingContext), new ArrayList<>()), 5L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.debug", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(loggingContext), new ArrayList<>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.warn", AggregationFunction.SUM, // mapreduce metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.MAPREDUCE, "mr1", Constants.Metrics.Tag.RUN_ID, "mrun1"), new ArrayList<>()), 2L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.trace", AggregationFunction.SUM, // workflow metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.WORKFLOW, "wflow1", Constants.Metrics.Tag.RUN_ID, "run1"), new ArrayList<>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.services.log.error", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<>()), 3L);
    } finally {
        pipeline.stopAndWait();
        loggerContext.stop();
        collectionService.stopAndWait();
    }
}

Example 9 with MetricStore

use of io.cdap.cdap.api.metrics.MetricStore in project cdap by cdapio.

the class MetricsProcessorServiceTest method testMetricsProcessor.

@Test
public void testMetricsProcessor() throws Exception {
    injector.getInstance(TransactionManager.class).startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    injector.getInstance(DatasetOpExecutorService.class).startAndWait();
    injector.getInstance(DatasetService.class).startAndWait();
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    Set<Integer> partitions = new HashSet<>();
    for (int i = 0; i < cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM); i++) {
        partitions.add(i);
    }
    // Start KafkaMetricsProcessorService after metrics are published to Kafka
    // Intentionally set queue size to a small value, so that MessagingMetricsProcessorManagerService
    // internally can persist metrics when more messages are to be fetched
    MessagingMetricsProcessorManagerService messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorManagerService.startAndWait();
    long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // Publish metrics with messaging service and record expected metrics
    for (int i = 10; i < 20; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.COUNTER);
    }
    Thread.sleep(500);
    // Stop and restart messagingMetricsProcessorManagerService
    messagingMetricsProcessorManagerService.stopAndWait();
    // Intentionally set queue size to a large value, so that MessagingMetricsProcessorManagerService
    // internally only persists metrics during terminating.
    messagingMetricsProcessorManagerService = new MessagingMetricsProcessorManagerService(cConf, injector.getInstance(MetricDatasetFactory.class), messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, injector.getInstance(MetricsWriterProvider.class), partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorManagerService.startAndWait();
    // Publish metrics after MessagingMetricsProcessorManagerService restarts and record expected metrics
    for (int i = 20; i < 30; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.GAUGE);
    }
    final List<String> missingMetricNames = new ArrayList<>();
    // are retrieved when timeout occurs, print out the missing metrics
    try {
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return canQueryAllMetrics(metricStore, METRICS_CONTEXT, expected, missingMetricNames);
            }
        }, 10000, TimeUnit.MILLISECONDS, "Failed to get all metrics");
    } catch (TimeoutException e) {
        Assert.fail(String.format("Metrics: [%s] cannot be found in the metrics store.", Joiner.on(", ").join(missingMetricNames)));
    }
    // Query metrics from the metricStore and compare them with the expected ones
    assertMetricsResult(metricStore, METRICS_CONTEXT, expected);
    // Query for the 5 counter metrics published with messaging between time 5 - 14
    Collection<MetricTimeSeries> queryResult = metricStore.query(new MetricDataQuery(5, 14, 1, Integer.MAX_VALUE, ImmutableMap.of(SYSTEM_METRIC_PREFIX + COUNTER_METRIC_NAME, AggregationFunction.SUM), METRICS_CONTEXT, ImmutableList.<String>of(), null));
    MetricTimeSeries timeSeries = Iterables.getOnlyElement(queryResult);
    Assert.assertEquals(5, timeSeries.getTimeValues().size());
    for (TimeValue timeValue : timeSeries.getTimeValues()) {
        Assert.assertEquals(1L, timeValue.getValue());
    }
    // Stop services and servers
    messagingMetricsProcessorManagerService.stopAndWait();
    // Delete all metrics
    metricStore.deleteAll();
}

Example 10 with MetricStore

use of io.cdap.cdap.api.metrics.MetricStore in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testMetricsEmit.

@Test
public void testMetricsEmit() throws Exception {
    ProfileService profileService = injector.getInstance(ProfileService.class);
    // create my profile
    ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
    profileService.saveProfile(myProfile, Profile.NATIVE);
    ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    RunId runId = RunIds.generate(System.currentTimeMillis());
    ProgramRunId programRunId = programId.run(runId.getId());
    Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
    long startTime = System.currentTimeMillis();
    // record the program status in app meta store
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        int sourceId = 0;
        metadataStoreDataset.recordProgramProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
        // using 3 nodes
        metadataStoreDataset.recordProgramProvisioned(programRunId, 3, AppFabricTestHelper.createSourceId(++sourceId));
        metadataStoreDataset.recordProgramStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
        metadataStoreDataset.recordProgramRunning(programRunId, startTime + 60000, null, AppFabricTestHelper.createSourceId(++sourceId));
    });
    programStateWriter.completed(programRunId);
    MetricStore metricStore = injector.getInstance(MetricStore.class);
    Map<String, String> additionalTags = getAdditionalTagsForProgramMetrics(ProgramRunStatus.RUNNING, null, ProgramRunClusterStatus.PROVISIONED);
    Tasks.waitFor(1L, () -> getMetric(metricStore, programRunId, myProfile, additionalTags, SYSTEM_METRIC_PREFIX + Constants.Metrics.Program.PROGRAM_COMPLETED_RUNS), 10, TimeUnit.SECONDS);
    // verify the metrics
    Assert.assertEquals(1L, getMetric(metricStore, programRunId, myProfile, additionalTags, SYSTEM_METRIC_PREFIX + Constants.Metrics.Program.PROGRAM_COMPLETED_RUNS));
    Assert.assertEquals(0L, getMetric(metricStore, programRunId, myProfile, additionalTags, SYSTEM_METRIC_PREFIX + Constants.Metrics.Program.PROGRAM_KILLED_RUNS));
    Assert.assertEquals(0L, getMetric(metricStore, programRunId, myProfile, additionalTags, SYSTEM_METRIC_PREFIX + Constants.Metrics.Program.PROGRAM_FAILED_RUNS));
    metricStore.deleteAll();
}