Examples with SystemProducer org.apache.samza.system.SystemProducer used on opensource projects

Example 1 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class DiagnosticsUtil method buildDiagnosticsManager.

/**
 * Create a {@link DiagnosticsManager} for the given jobName, jobId, containerId, and execEnvContainerId, if
 * diagnostics is enabled.
 * @param executionEnvContainerId ID assigned to the container by the cluster manager (e.g. YARN)
 * @param samzaEpochId ID assigned to the job deployment attempt by the cluster manager
 */
public static Optional<DiagnosticsManager> buildDiagnosticsManager(String jobName, String jobId, JobModel jobModel, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, Config config) {
    JobConfig jobConfig = new JobConfig(config);
    MetricsConfig metricsConfig = new MetricsConfig(config);
    Optional<DiagnosticsManager> diagnosticsManagerOptional = Optional.empty();
    if (jobConfig.getDiagnosticsEnabled()) {
        ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
        int containerMemoryMb = clusterManagerConfig.getContainerMemoryMb();
        int containerNumCores = clusterManagerConfig.getNumCores();
        long maxHeapSizeBytes = Runtime.getRuntime().maxMemory();
        int containerThreadPoolSize = jobConfig.getThreadPoolSize();
        String taskClassVersion = Util.getTaskClassVersion(config);
        String samzaVersion = Util.getSamzaVersion();
        String hostName = Util.getLocalHost().getHostName();
        Optional<String> diagnosticsReporterStreamName = metricsConfig.getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS);
        if (!diagnosticsReporterStreamName.isPresent()) {
            throw new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS));
        }
        SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsReporterStreamName.get());
        // Create a SystemProducer for DiagnosticsManager. This producer is used by the DiagnosticsManager
        // to write to the same stream as the MetricsSnapshotReporter called `diagnosticsreporter`.
        Optional<String> diagnosticsSystemFactoryName = new SystemConfig(config).getSystemFactory(diagnosticsSystemStream.getSystem());
        if (!diagnosticsSystemFactoryName.isPresent()) {
            throw new SamzaException("Missing factory in config for system " + diagnosticsSystemStream.getSystem());
        }
        SystemFactory systemFactory = ReflectionUtil.getObj(diagnosticsSystemFactoryName.get(), SystemFactory.class);
        SystemProducer systemProducer = systemFactory.getProducer(diagnosticsSystemStream.getSystem(), config, new MetricsRegistryMap(), DiagnosticsUtil.class.getSimpleName());
        DiagnosticsManager diagnosticsManager = new DiagnosticsManager(jobName, jobId, jobModel.getContainers(), containerMemoryMb, containerNumCores, new StorageConfig(config).getNumPersistentStores(), maxHeapSizeBytes, containerThreadPoolSize, containerId, executionEnvContainerId.orElse(""), samzaEpochId.orElse(""), taskClassVersion, samzaVersion, hostName, diagnosticsSystemStream, systemProducer, Duration.ofMillis(new TaskConfig(config).getShutdownMs()), jobConfig.getAutosizingEnabled(), config);
        diagnosticsManagerOptional = Optional.of(diagnosticsManager);
    }
    return diagnosticsManagerOptional;
}

Example 2 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestInMemorySystem method produceMessages.

private void produceMessages(Object... events) {
    SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry);
    Stream.of(events).forEach(event -> systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, event)));
}

Example 3 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestDiagnosticsUtil method testBuildDiagnosticsManager.

@Test
public void testBuildDiagnosticsManager() {
    Config config = new MapConfig(buildTestConfigs());
    JobModel mockJobModel = mock(JobModel.class);
    SystemFactory systemFactory = mock(SystemFactory.class);
    SystemProducer mockProducer = mock(SystemProducer.class);
    when(systemFactory.getProducer(anyString(), any(Config.class), any(MetricsRegistry.class), anyString())).thenReturn(mockProducer);
    PowerMockito.mockStatic(ReflectionUtil.class);
    when(ReflectionUtil.getObj(SYSTEM_FACTORY, SystemFactory.class)).thenReturn(systemFactory);
    Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(JOB_NAME, JOB_ID, mockJobModel, CONTAINER_ID, Optional.of(ENV_ID), Optional.of(SAMZA_EPOCH_ID), config);
    Assert.assertTrue(diagnosticsManager.isPresent());
}

Example 4 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class MetricsSnapshotReporterFactory method getMetricsReporter.

@Override
public MetricsReporter getMetricsReporter(String reporterName, String containerName, Config config) {
    LOG.info("Creating new metrics snapshot reporter.");
    MetricsRegistryMap registry = new MetricsRegistryMap();
    SystemStream systemStream = getSystemStream(reporterName, config);
    SystemProducer producer = getProducer(reporterName, config, registry);
    Duration reportingInterval = Duration.ofSeconds(getReportingInterval(reporterName, config));
    String jobName = getJobName(config);
    String jobId = getJobId(config);
    Serde<MetricsSnapshot> serde = getSerde(reporterName, config);
    Optional<Pattern> blacklist = getBlacklist(reporterName, config);
    MetricsSnapshotReporter reporter = new MetricsSnapshotReporter(producer, systemStream, reportingInterval, jobName, jobId, containerName, Util.getTaskClassVersion(config), Util.getSamzaVersion(), Util.getLocalHost().getHostName(), serde, blacklist, SystemClock.instance());
    reporter.register(this.getClass().getSimpleName(), registry);
    return reporter;
}

Example 5 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestDiagnosticsManager method testDiagnosticsManagerStop.

@Test
public void testDiagnosticsManagerStop() throws InterruptedException {
    SystemProducer mockSystemProducer = Mockito.mock(SystemProducer.class);
    Mockito.when(mockExecutorService.isTerminated()).thenReturn(true);
    Duration terminationDuration = Duration.ofSeconds(1);
    DiagnosticsManager diagnosticsManager = new DiagnosticsManager(JOB_NAME, JOB_ID, containerModels, CONTAINER_MB, CONTAINER_NUM_CORES, NUM_PERSISTENT_STORES, MAX_HEAP_SIZE, CONTAINER_THREAD_POOL_SIZE, "0", EXECUTION_ENV_CONTAINER_ID, SAMZA_EPOCH_ID, TASK_CLASS_VERSION, SAMZA_VERSION, HOSTNAME, diagnosticsSystemStream, mockSystemProducer, terminationDuration, mockExecutorService, AUTOSIZING_ENABLED, config, this.clock);
    diagnosticsManager.stop();
    Mockito.verify(mockExecutorService, Mockito.times(1)).shutdown();
    Mockito.verify(mockExecutorService, Mockito.times(1)).awaitTermination(terminationDuration.toMillis(), TimeUnit.MILLISECONDS);
    Mockito.verify(mockExecutorService, Mockito.never()).shutdownNow();
    Mockito.verify(mockSystemProducer, Mockito.times(1)).stop();
}