Examples with MetricsCollectionService io.cdap.cdap.api.metrics.MetricsCollectionService used on opensource projects

Example 6 with MetricsCollectionService

use of io.cdap.cdap.api.metrics.MetricsCollectionService in project cdap by caskdata.

the class ProfileMetricServiceTest method testProfileMetrics.

@Test
public void testProfileMetrics() throws Exception {
    ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate());
    ProfileId profileId = NamespaceId.DEFAULT.profile("myProfile");
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    MetricStore metricStore = injector.getInstance(MetricStore.class);
    // There are 5 nodes, we emit the metrics each 2 mins, so each time the node minute should go up by 10 min
    ProfileMetricService scheduledService = new ProfileMetricService(collectionService, runId, profileId, 5, 2);
    // emit and verify the results
    scheduledService.emitMetric();
    Tasks.waitFor(10L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.emitMetric();
    Tasks.waitFor(20L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.emitMetric();
    Tasks.waitFor(30L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
}

Example 7 with MetricsCollectionService

use of io.cdap.cdap.api.metrics.MetricsCollectionService in project cdap by caskdata.

the class ProfileMetricServiceTest method testRoundingLogic.

@Test
public void testRoundingLogic() throws Exception {
    ProgramRunId runId = NamespaceId.DEFAULT.app("round").workflow("round").run(RunIds.generate());
    ProfileId profileId = NamespaceId.DEFAULT.profile("roundProfile");
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    MetricStore metricStore = injector.getInstance(MetricStore.class);
    ProfileMetricService scheduledService = new ProfileMetricService(collectionService, runId, profileId, 1, 1);
    // start and stop the service, the metric should still go up by 1
    scheduledService.startUp();
    scheduledService.shutDown();
    Tasks.waitFor(1L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.startUp();
    // set the start up time to 90 seconds before the current time
    scheduledService.setStartUpTime(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) - 90);
    // 90 seconds should round up to 2 mins, so emit 1 min and test the rounding logic
    scheduledService.emitMetric();
    scheduledService.shutDown();
    // the metric should go up by 2
    Tasks.waitFor(3L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
    scheduledService.startUp();
    // set the start up time to 65 seconds before the current time
    scheduledService.setStartUpTime(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) - 65);
    // 65 seconds should round down to 1 min, so emit 1 min and test the rest seconds are ignored
    scheduledService.emitMetric();
    scheduledService.shutDown();
    // the metric should go up by 1
    Tasks.waitFor(4L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS);
}

Example 8 with MetricsCollectionService

use of io.cdap.cdap.api.metrics.MetricsCollectionService in project cdap by caskdata.

the class ProgramStatusEventPublisherMetricsTest method testMetrics.

@Test
public void testMetrics() throws IOException {
    MetricsProvider mockMetricsProvider = getMockMetricsProvider();
    List<MetricValues> metricValuesList = new ArrayList<>();
    MetricsCollectionService mockMetricsCollectionService = getMockCollectionService(metricValuesList);
    mockMetricsCollectionService.startAndWait();
    ProgramStatusEventPublisher programStatusEventPublisher = new ProgramStatusEventPublisher(CConfiguration.create(), null, mockMetricsCollectionService, null, mockMetricsProvider);
    programStatusEventPublisher.initialize(Collections.singleton(new DummyEventWriter()));
    programStatusEventPublisher.processMessages(null, getMockNotification());
    mockMetricsCollectionService.stopAndWait();
    Assert.assertSame(1, metricValuesList.size());
    Assert.assertTrue(containsMetric(metricValuesList.get(0), Constants.Metrics.ProgramEvent.PUBLISHED_COUNT));
}

Example 9 with MetricsCollectionService

use of io.cdap.cdap.api.metrics.MetricsCollectionService in project cdap by caskdata.

the class MetricsReporterHookTest method testReponseTimeCollection.

@Test
public void testReponseTimeCollection() throws InterruptedException {
    MetricsContext mockCollector = mock(MetricsContext.class);
    MetricsCollectionService mockCollectionService = mock(MetricsCollectionService.class);
    when(mockCollectionService.getContext(anyMap())).thenReturn(mockCollector);
    HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "http://ignore");
    HandlerInfo handlerInfo = new HandlerInfo(TESTHANDLERNAME, TESTMETHODNAME);
    MetricsReporterHook hook = new MetricsReporterHook(CConfiguration.create(), mockCollectionService, TESTSERVICENAME);
    hook.preCall(request, null, handlerInfo);
    hook.postCall(request, HttpResponseStatus.OK, handlerInfo);
    verify(mockCollector).event(eq("response.latency"), anyLong());
}

Example 10 with MetricsCollectionService

use of io.cdap.cdap.api.metrics.MetricsCollectionService in project cdap by caskdata.

the class SparkProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    LOG.trace("Starting Spark program {} with SparkProgramRunner of ClassLoader {}", program.getId(), getClass().getClassLoader());
    // Get the RunId first. It is used for the creation of the ClassLoader closing thread.
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    Deque<Closeable> closeables = new LinkedList<>();
    try {
        // Extract and verify parameters
        ApplicationSpecification appSpec = program.getApplicationSpecification();
        Preconditions.checkNotNull(appSpec, "Missing application specification.");
        ProgramType processorType = program.getType();
        Preconditions.checkNotNull(processorType, "Missing processor type.");
        Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only Spark process type is supported.");
        SparkSpecification spec = appSpec.getSpark().get(program.getName());
        Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
        String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
        Preconditions.checkArgument(host != null, "No hostname is provided");
        // Get the WorkflowProgramInfo if it is started by Workflow
        WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ProgramId programId = program.getId();
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.addFirst(pluginInstantiator);
        }
        SparkRuntimeContext runtimeContext = new SparkRuntimeContext(new Configuration(hConf), program, options, cConf, host, txClient, programDatasetFramework, metricsCollectionService, workflowInfo, pluginInstantiator, secureStore, secureStoreManager, accessEnforcer, authenticationContext, messagingService, serviceAnnouncer, pluginFinder, locationFactory, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory, () -> {
        });
        closeables.addFirst(runtimeContext);
        Spark spark;
        try {
            spark = new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
        } catch (Exception e) {
            LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
            throw Throwables.propagate(e);
        }
        boolean isLocal = SparkRuntimeContextConfig.isLocal(options);
        SparkSubmitter submitter;
        // If MasterEnvironment is not available, use non-master env spark submitters
        MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
        if (masterEnv != null && cConf.getBoolean(Constants.Environment.PROGRAM_SUBMISSION_MASTER_ENV_ENABLED, true)) {
            submitter = new MasterEnvironmentSparkSubmitter(cConf, locationFactory, host, runtimeContext, masterEnv, options);
        } else {
            submitter = isLocal ? new LocalSparkSubmitter() : new DistributedSparkSubmitter(hConf, locationFactory, host, runtimeContext, options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE));
        }
        Service sparkRuntimeService = new SparkRuntimeService(cConf, spark, getPluginArchive(options), runtimeContext, submitter, locationFactory, isLocal, fieldLineageWriter, masterEnv, commonNettyHttpServiceFactory);
        sparkRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new SparkProgramController(sparkRuntimeService, runtimeContext);
        LOG.debug("Starting Spark Job. Context: {}", runtimeContext);
        if (isLocal || UserGroupInformation.isSecurityEnabled()) {
            sparkRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService);
        }
        return controller;
    } catch (Throwable t) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(t);
    }
}