use of org.apache.samza.config.MetricsConfig in project samza by apache.
the class DiagnosticsUtil method createDiagnosticsStream.
public static void createDiagnosticsStream(Config config) {
if (!new JobConfig(config).getDiagnosticsEnabled()) {
return;
}
// if diagnostics is enabled, create diagnostics stream if it doesnt exist
String diagnosticsSystemStreamName = new MetricsConfig(config).getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS).orElseThrow(() -> new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)));
SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName);
SystemConfig systemConfig = new SystemConfig(config);
SystemAdmin diagnosticsSysAdmin = systemConfig.getSystemFactories().get(diagnosticsSystemStream.getSystem()).getAdmin(diagnosticsSystemStream.getSystem(), config, DiagnosticsUtil.class.getSimpleName());
StreamSpec diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream(), diagnosticsSystemStream.getSystem(), new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID));
log.info("Creating diagnostics stream {}", diagnosticsSystemStream.getStream());
diagnosticsSysAdmin.start();
if (diagnosticsSysAdmin.createStream(diagnosticsStreamSpec)) {
log.info("Created diagnostics stream {}", diagnosticsSystemStream.getStream());
} else {
log.info("Diagnostics stream {} already exists", diagnosticsSystemStream.getStream());
}
diagnosticsSysAdmin.stop();
}
use of org.apache.samza.config.MetricsConfig in project samza by apache.
the class DiagnosticsUtil method buildDiagnosticsManager.
/**
* Create a {@link DiagnosticsManager} for the given jobName, jobId, containerId, and execEnvContainerId, if
* diagnostics is enabled.
* @param executionEnvContainerId ID assigned to the container by the cluster manager (e.g. YARN)
* @param samzaEpochId ID assigned to the job deployment attempt by the cluster manager
*/
public static Optional<DiagnosticsManager> buildDiagnosticsManager(String jobName, String jobId, JobModel jobModel, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, Config config) {
JobConfig jobConfig = new JobConfig(config);
MetricsConfig metricsConfig = new MetricsConfig(config);
Optional<DiagnosticsManager> diagnosticsManagerOptional = Optional.empty();
if (jobConfig.getDiagnosticsEnabled()) {
ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
int containerMemoryMb = clusterManagerConfig.getContainerMemoryMb();
int containerNumCores = clusterManagerConfig.getNumCores();
long maxHeapSizeBytes = Runtime.getRuntime().maxMemory();
int containerThreadPoolSize = jobConfig.getThreadPoolSize();
String taskClassVersion = Util.getTaskClassVersion(config);
String samzaVersion = Util.getSamzaVersion();
String hostName = Util.getLocalHost().getHostName();
Optional<String> diagnosticsReporterStreamName = metricsConfig.getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS);
if (!diagnosticsReporterStreamName.isPresent()) {
throw new ConfigException("Missing required config: " + String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM, MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS));
}
SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsReporterStreamName.get());
// Create a SystemProducer for DiagnosticsManager. This producer is used by the DiagnosticsManager
// to write to the same stream as the MetricsSnapshotReporter called `diagnosticsreporter`.
Optional<String> diagnosticsSystemFactoryName = new SystemConfig(config).getSystemFactory(diagnosticsSystemStream.getSystem());
if (!diagnosticsSystemFactoryName.isPresent()) {
throw new SamzaException("Missing factory in config for system " + diagnosticsSystemStream.getSystem());
}
SystemFactory systemFactory = ReflectionUtil.getObj(diagnosticsSystemFactoryName.get(), SystemFactory.class);
SystemProducer systemProducer = systemFactory.getProducer(diagnosticsSystemStream.getSystem(), config, new MetricsRegistryMap(), DiagnosticsUtil.class.getSimpleName());
DiagnosticsManager diagnosticsManager = new DiagnosticsManager(jobName, jobId, jobModel.getContainers(), containerMemoryMb, containerNumCores, new StorageConfig(config).getNumPersistentStores(), maxHeapSizeBytes, containerThreadPoolSize, containerId, executionEnvContainerId.orElse(""), samzaEpochId.orElse(""), taskClassVersion, samzaVersion, hostName, diagnosticsSystemStream, systemProducer, Duration.ofMillis(new TaskConfig(config).getShutdownMs()), jobConfig.getAutosizingEnabled(), config);
diagnosticsManagerOptional = Optional.of(diagnosticsManager);
}
return diagnosticsManagerOptional;
}
use of org.apache.samza.config.MetricsConfig in project samza by apache.
the class BaseReadWriteUpdateTable method init.
@Override
public void init(Context context) {
MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig());
clock = metricsConfig.getMetricsTimerEnabled() ? () -> System.nanoTime() : () -> 0L;
metrics = new TableMetrics(context, this, tableId);
}
use of org.apache.samza.config.MetricsConfig in project samza by apache.
the class AsyncRateLimitedTable method init.
@Override
public void init(Context context) {
table.init(context);
MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig());
if (metricsConfig.getMetricsTimerEnabled()) {
TableMetricsUtil tableMetricsUtil = new TableMetricsUtil(context, this, tableId);
if (isReadRateLimited()) {
readRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("get-throttle-ns"));
}
if (isWriteRateLimited()) {
writeRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("put-throttle-ns"));
}
if (isUpdateRateLimited()) {
updateRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("update-throttle-ns"));
}
}
}
use of org.apache.samza.config.MetricsConfig in project samza by apache.
the class JobCoordinatorLaunchUtil method runJobCoordinator.
private static void runJobCoordinator(String jobCoordinatorClassName, MetricsRegistryMap metrics, MetadataStore metadataStore, Config finalConfig) {
JobCoordinatorFactory jobCoordinatorFactory = ReflectionUtil.getObj(jobCoordinatorClassName, JobCoordinatorFactory.class);
JobCoordinator jobCoordinator = jobCoordinatorFactory.getJobCoordinator(JOB_COORDINATOR_PROCESSOR_ID_PLACEHOLDER, finalConfig, metrics, metadataStore);
Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(finalConfig), CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME);
metricsReporters.values().forEach(metricsReporter -> metricsReporter.register(CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, metrics));
metricsReporters.values().forEach(MetricsReporter::start);
CountDownLatch waitForShutdownLatch = new CountDownLatch(1);
jobCoordinator.setListener(new NoProcessorJobCoordinatorListener(waitForShutdownLatch));
jobCoordinator.start();
addShutdownHook(jobCoordinator);
try {
waitForShutdownLatch.await();
} catch (InterruptedException e) {
String errorMessage = "Error while waiting for coordinator to complete";
LOG.error(errorMessage, e);
throw new SamzaException(errorMessage, e);
} finally {
metricsReporters.values().forEach(MetricsReporter::stop);
}
}
Aggregations