Examples with MetricsConfig org.apache.samza.config.MetricsConfig used on opensource projects

Example 1 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class DiagnosticsUtil method createDiagnosticsStream.

public static void createDiagnosticsStream(Config config) {
    if (!new JobConfig(config).getDiagnosticsEnabled()) {
        return;
    }
    // if diagnostics is enabled, create diagnostics stream if it doesnt exist
    String diagnosticsSystemStreamName = new MetricsConfig(config).getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS).orElseThrow(() -> new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)));
    SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName);
    SystemConfig systemConfig = new SystemConfig(config);
    SystemAdmin diagnosticsSysAdmin = systemConfig.getSystemFactories().get(diagnosticsSystemStream.getSystem()).getAdmin(diagnosticsSystemStream.getSystem(), config, DiagnosticsUtil.class.getSimpleName());
    StreamSpec diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream(), diagnosticsSystemStream.getSystem(), new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID));
    log.info("Creating diagnostics stream {}", diagnosticsSystemStream.getStream());
    diagnosticsSysAdmin.start();
    if (diagnosticsSysAdmin.createStream(diagnosticsStreamSpec)) {
        log.info("Created diagnostics stream {}", diagnosticsSystemStream.getStream());
    } else {
        log.info("Diagnostics stream {} already exists", diagnosticsSystemStream.getStream());
    }
    diagnosticsSysAdmin.stop();
}

Example 2 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class DiagnosticsUtil method buildDiagnosticsManager.

/**
 * Create a {@link DiagnosticsManager} for the given jobName, jobId, containerId, and execEnvContainerId, if
 * diagnostics is enabled.
 * @param executionEnvContainerId ID assigned to the container by the cluster manager (e.g. YARN)
 * @param samzaEpochId ID assigned to the job deployment attempt by the cluster manager
 */
public static Optional<DiagnosticsManager> buildDiagnosticsManager(String jobName, String jobId, JobModel jobModel, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, Config config) {
    JobConfig jobConfig = new JobConfig(config);
    MetricsConfig metricsConfig = new MetricsConfig(config);
    Optional<DiagnosticsManager> diagnosticsManagerOptional = Optional.empty();
    if (jobConfig.getDiagnosticsEnabled()) {
        ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
        int containerMemoryMb = clusterManagerConfig.getContainerMemoryMb();
        int containerNumCores = clusterManagerConfig.getNumCores();
        long maxHeapSizeBytes = Runtime.getRuntime().maxMemory();
        int containerThreadPoolSize = jobConfig.getThreadPoolSize();
        String taskClassVersion = Util.getTaskClassVersion(config);
        String samzaVersion = Util.getSamzaVersion();
        String hostName = Util.getLocalHost().getHostName();
        Optional<String> diagnosticsReporterStreamName = metricsConfig.getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS);
        if (!diagnosticsReporterStreamName.isPresent()) {
            throw new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS));
        }
        SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsReporterStreamName.get());
        // Create a SystemProducer for DiagnosticsManager. This producer is used by the DiagnosticsManager
        // to write to the same stream as the MetricsSnapshotReporter called `diagnosticsreporter`.
        Optional<String> diagnosticsSystemFactoryName = new SystemConfig(config).getSystemFactory(diagnosticsSystemStream.getSystem());
        if (!diagnosticsSystemFactoryName.isPresent()) {
            throw new SamzaException("Missing factory in config for system " + diagnosticsSystemStream.getSystem());
        }
        SystemFactory systemFactory = ReflectionUtil.getObj(diagnosticsSystemFactoryName.get(), SystemFactory.class);
        SystemProducer systemProducer = systemFactory.getProducer(diagnosticsSystemStream.getSystem(), config, new MetricsRegistryMap(), DiagnosticsUtil.class.getSimpleName());
        DiagnosticsManager diagnosticsManager = new DiagnosticsManager(jobName, jobId, jobModel.getContainers(), containerMemoryMb, containerNumCores, new StorageConfig(config).getNumPersistentStores(), maxHeapSizeBytes, containerThreadPoolSize, containerId, executionEnvContainerId.orElse(""), samzaEpochId.orElse(""), taskClassVersion, samzaVersion, hostName, diagnosticsSystemStream, systemProducer, Duration.ofMillis(new TaskConfig(config).getShutdownMs()), jobConfig.getAutosizingEnabled(), config);
        diagnosticsManagerOptional = Optional.of(diagnosticsManager);
    }
    return diagnosticsManagerOptional;
}

Example 3 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class BaseReadWriteUpdateTable method init.

@Override
public void init(Context context) {
    MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig());
    clock = metricsConfig.getMetricsTimerEnabled() ? () -> System.nanoTime() : () -> 0L;
    metrics = new TableMetrics(context, this, tableId);
}

Example 4 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class AsyncRateLimitedTable method init.

@Override
public void init(Context context) {
    table.init(context);
    MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig());
    if (metricsConfig.getMetricsTimerEnabled()) {
        TableMetricsUtil tableMetricsUtil = new TableMetricsUtil(context, this, tableId);
        if (isReadRateLimited()) {
            readRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("get-throttle-ns"));
        }
        if (isWriteRateLimited()) {
            writeRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("put-throttle-ns"));
        }
        if (isUpdateRateLimited()) {
            updateRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("update-throttle-ns"));
        }
    }
}

Example 5 with MetricsConfig

use of org.apache.samza.config.MetricsConfig in project samza by apache.

the class JobCoordinatorLaunchUtil method runJobCoordinator.

private static void runJobCoordinator(String jobCoordinatorClassName, MetricsRegistryMap metrics, MetadataStore metadataStore, Config finalConfig) {
    JobCoordinatorFactory jobCoordinatorFactory = ReflectionUtil.getObj(jobCoordinatorClassName, JobCoordinatorFactory.class);
    JobCoordinator jobCoordinator = jobCoordinatorFactory.getJobCoordinator(JOB_COORDINATOR_PROCESSOR_ID_PLACEHOLDER, finalConfig, metrics, metadataStore);
    Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(finalConfig), CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME);
    metricsReporters.values().forEach(metricsReporter -> metricsReporter.register(CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, metrics));
    metricsReporters.values().forEach(MetricsReporter::start);
    CountDownLatch waitForShutdownLatch = new CountDownLatch(1);
    jobCoordinator.setListener(new NoProcessorJobCoordinatorListener(waitForShutdownLatch));
    jobCoordinator.start();
    addShutdownHook(jobCoordinator);
    try {
        waitForShutdownLatch.await();
    } catch (InterruptedException e) {
        String errorMessage = "Error while waiting for coordinator to complete";
        LOG.error(errorMessage, e);
        throw new SamzaException(errorMessage, e);
    } finally {
        metricsReporters.values().forEach(MetricsReporter::stop);
    }
}