Examples with MetricsConfig org.apache.samza.config.MetricsConfig used on opensource projects

Example 6 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class TestJobCoordinatorLaunchUtil method testRunJobCoordinator.

@Test
public void testRunJobCoordinator() throws Exception {
    String jobCoordinatorFactoryClass = "org.apache.samza.custom.MyJobCoordinatorFactory";
    Config originalConfig = buildOriginalConfig(ImmutableMap.of(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, jobCoordinatorFactoryClass));
    JobConfig fullConfig = new JobConfig(new MapConfig(originalConfig, Collections.singletonMap("isAfterPlanning", "true")));
    Config autoSizingConfig = new MapConfig(Collections.singletonMap(JobConfig.JOB_AUTOSIZING_CONTAINER_COUNT, "10"));
    Config finalConfig = new MapConfig(autoSizingConfig, fullConfig);
    RemoteJobPlanner remoteJobPlanner = mock(RemoteJobPlanner.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    JobCoordinatorFactory jobCoordinatorFactory = mock(JobCoordinatorFactory.class);
    JobCoordinator jobCoordinator = mock(JobCoordinator.class);
    PowerMockito.mockStatic(CoordinatorStreamUtil.class);
    PowerMockito.doNothing().when(CoordinatorStreamUtil.class, "createCoordinatorStream", any());
    PowerMockito.doReturn(new MapConfig()).when(CoordinatorStreamUtil.class, "buildCoordinatorStreamConfig", any());
    PowerMockito.doReturn(autoSizingConfig).when(CoordinatorStreamUtil.class, "readLaunchConfigFromCoordinatorStream", any(), any());
    PowerMockito.whenNew(CoordinatorStreamStore.class).withAnyArguments().thenReturn(coordinatorStreamStore);
    PowerMockito.whenNew(RemoteJobPlanner.class).withAnyArguments().thenReturn(remoteJobPlanner);
    when(remoteJobPlanner.prepareJobs()).thenReturn(Collections.singletonList(fullConfig));
    PowerMockito.mockStatic(ReflectionUtil.class);
    PowerMockito.doReturn(jobCoordinatorFactory).when(ReflectionUtil.class, "getObj", jobCoordinatorFactoryClass, JobCoordinatorFactory.class);
    when(jobCoordinatorFactory.getJobCoordinator(eq("samza-job-coordinator"), eq(finalConfig), any(), eq(coordinatorStreamStore))).thenReturn(jobCoordinator);
    // use a latch to keep track of when shutdown hook was added to know when we should start verifications
    CountDownLatch addShutdownHookLatch = new CountDownLatch(1);
    PowerMockito.spy(JobCoordinatorLaunchUtil.class);
    PowerMockito.doAnswer(invocation -> {
        addShutdownHookLatch.countDown();
        return null;
    }).when(JobCoordinatorLaunchUtil.class, "addShutdownHook", any());
    MetricsReporter metricsReporter = mock(MetricsReporter.class);
    Map<String, MetricsReporter> metricsReporterMap = ImmutableMap.of("reporter", metricsReporter);
    PowerMockito.mockStatic(MetricsReporterLoader.class);
    PowerMockito.doReturn(metricsReporterMap).when(MetricsReporterLoader.class, "getMetricsReporters", new MetricsConfig(finalConfig), "JobCoordinator");
    NoProcessorJobCoordinatorListener jobCoordinatorListener = mock(NoProcessorJobCoordinatorListener.class);
    PowerMockito.whenNew(NoProcessorJobCoordinatorListener.class).withAnyArguments().thenReturn(jobCoordinatorListener);
    Thread runThread = new Thread(() -> JobCoordinatorLaunchUtil.run(new MockStreamApplication(), originalConfig));
    runThread.start();
    // last thing before waiting for shutdown is to add shutdown hook, so do verifications once hook is added
    addShutdownHookLatch.await();
    verifyStatic();
    CoordinatorStreamUtil.createCoordinatorStream(fullConfig);
    verifyStatic();
    CoordinatorStreamUtil.writeConfigToCoordinatorStream(finalConfig, true);
    verifyStatic();
    JobCoordinatorLaunchUtil.addShutdownHook(jobCoordinator);
    InOrder inOrder = Mockito.inOrder(metricsReporter, jobCoordinator);
    inOrder.verify(metricsReporter).register(eq("JobCoordinator"), any());
    inOrder.verify(metricsReporter).start();
    ArgumentCaptor<CountDownLatch> countDownLatchArgumentCaptor = ArgumentCaptor.forClass(CountDownLatch.class);
    verifyNew(NoProcessorJobCoordinatorListener.class).withArguments(countDownLatchArgumentCaptor.capture());
    inOrder.verify(jobCoordinator).setListener(jobCoordinatorListener);
    inOrder.verify(jobCoordinator).start();
    // wait some time and then make sure the run thread is still alive
    Thread.sleep(Duration.ofMillis(500).toMillis());
    assertTrue(runThread.isAlive());
    // trigger the count down latch so that the run thread can exit
    countDownLatchArgumentCaptor.getValue().countDown();
    runThread.join(Duration.ofSeconds(10).toMillis());
    assertFalse(runThread.isAlive());
    verify(metricsReporter).stop();
}

Example 7 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class MetricsSnapshotReporterFactory method getSystemStream.

protected SystemStream getSystemStream(String reporterName, Config config) {
    MetricsConfig metricsConfig = new MetricsConfig(config);
    String metricsSystemStreamName = metricsConfig.getMetricsSnapshotReporterStream(reporterName).orElseThrow(() -> new SamzaException("No metrics stream defined in config."));
    SystemStream systemStream = StreamUtil.getSystemStreamFromNames(metricsSystemStreamName);
    LOG.info("Got system stream {}.", systemStream);
    return systemStream;
}

Example 8 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class MetricsSnapshotReporterFactory method getBlacklist.

protected Optional<Pattern> getBlacklist(String reporterName, Config config) {
    MetricsConfig metricsConfig = new MetricsConfig(config);
    Optional<String> blacklist = metricsConfig.getMetricsSnapshotReporterBlacklist(reporterName);
    LOG.info("Got blacklist as: {}", blacklist);
    return blacklist.map(Pattern::compile);
}

Example 9 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class StreamAppender method setupSystem.

/**
 * This should only be called after verifying that the {@link LoggingContextHolder} has the config.
 */
protected void setupSystem() {
    config = getConfig();
    Log4jSystemConfig log4jSystemConfig = getLog4jSystemConfig(config);
    if (streamName == null) {
        streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
    }
    // Instantiate metrics
    MetricsRegistryMap metricsRegistry = new MetricsRegistryMap();
    // Take this.getClass().getName() as the name to make it extend-friendly
    metrics = getMetrics(metricsRegistry);
    // Register metrics into metrics reporters so that they are able to be reported to other systems
    Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), containerName);
    metricsReporters.values().forEach(reporter -> {
        reporter.register(containerName, metricsRegistry);
        reporter.start();
    });
    String systemName = log4jSystemConfig.getSystemName();
    String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName).orElseThrow(() -> new SamzaException("Could not figure out \"" + systemName + "\" system factory for log4j " + getName() + " to use"));
    SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);
    setSerde(log4jSystemConfig, systemName);
    setupStream(systemFactory, systemName);
    systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry, this.getClass().getSimpleName());
    systemStream = new SystemStream(systemName, streamName);
    systemProducer.register(SOURCE);
    systemProducer.start();
    System.out.println(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName + " in " + systemName + ". Logs are partitioned by " + key);
    startTransferThread();
}

Example 10 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class SamzaRestService method main.

/**
 * Command line interface to run the server.
 *
 * @param args arguments supported by {@link org.apache.samza.util.CommandLine}.
 *             In particular, --config job.config.loader.properties.path and
 *             --config job.config.loader.factory are used to read the Samza REST config file.
 * @throws Exception if the server could not be successfully started.
 */
public static void main(String[] args) throws Exception {
    SamzaMonitorService monitorService = null;
    try {
        SamzaRestConfig config = parseConfig(args);
        ReadableMetricsRegistry metricsRegistry = new MetricsRegistryMap();
        log.info("Creating new SamzaRestService with config: {}", config);
        MetricsConfig metricsConfig = new MetricsConfig(config);
        Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(metricsConfig, Util.getLocalHost().getHostName());
        SamzaRestService restService = new SamzaRestService(new Server(config.getPort()), metricsRegistry, metricsReporters, new ServletContextHandler(ServletContextHandler.SESSIONS));
        // Add applications
        SamzaRestApplication samzaRestApplication = new SamzaRestApplication(config);
        ServletContainer container = new ServletContainer(samzaRestApplication);
        restService.addServlet(container, "/*");
        monitorService = new SamzaMonitorService(config, metricsRegistry);
        monitorService.start();
        restService.runBlocking();
    } catch (Throwable t) {
        log.error("Exception in main.", t);
    } finally {
        if (monitorService != null) {
            monitorService.stop();
        }
    }
}