Search in sources :

Example 66 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class ContainerHeartbeatMonitor method initializeMetrics.

private void initializeMetrics(Config config) {
    reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), SOURCE_NAME);
    MetricsRegistryMap registryMap = new MetricsRegistryMap();
    metrics = new ContainerHeartbeatMetrics(registryMap);
    reporters.values().forEach(reporter -> reporter.register(SOURCE_NAME, registryMap));
}
Also used : MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) MetricsConfig(org.apache.samza.config.MetricsConfig)

Example 67 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class SSPGrouperProxy method getSystemStreamPartitionMapper.

/**
 * Creates a instance of {@link SystemStreamPartitionMapper} using the stream partition expansion factory class
 * defined in the {@param config}.
 * @param config the configuration of the samza job.
 * @return the instantiated {@link SystemStreamPartitionMapper} object.
 */
private SystemStreamPartitionMapper getSystemStreamPartitionMapper(Config config) {
    JobConfig jobConfig = new JobConfig(config);
    String systemStreamPartitionMapperClass = jobConfig.getSystemStreamPartitionMapperFactoryName();
    SystemStreamPartitionMapperFactory systemStreamPartitionMapperFactory = ReflectionUtil.getObj(systemStreamPartitionMapperClass, SystemStreamPartitionMapperFactory.class);
    return systemStreamPartitionMapperFactory.getStreamPartitionMapper(config, new MetricsRegistryMap());
}
Also used : MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) JobConfig(org.apache.samza.config.JobConfig)

Example 68 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class SamzaTaskProxy method readTasksFromCoordinatorStream.

/**
 * Builds list of {@link Task} from job model in coordinator stream.
 * @param consumer system consumer associated with a job's coordinator stream.
 * @return list of {@link Task} constructed from job model in coordinator stream.
 */
protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) {
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(consumer.getConfig(), new MetricsRegistryMap());
    LocalityManager localityManager = new LocalityManager(coordinatorStreamStore);
    Map<String, ProcessorLocality> containerLocalities = localityManager.readLocality().getProcessorLocalities();
    TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetTaskContainerMapping.TYPE), new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetTaskModeMapping.TYPE));
    Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment();
    StorageConfig storageConfig = new StorageConfig(consumer.getConfig());
    List<String> storeNames = storageConfig.getStoreNames();
    return taskNameToContainerIdMapping.entrySet().stream().map(entry -> {
        String hostName = Optional.ofNullable(containerLocalities.get(entry.getValue())).map(ProcessorLocality::host).orElse(null);
        return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames);
    }).collect(Collectors.toList());
}
Also used : CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) SetTaskModeMapping(org.apache.samza.coordinator.stream.messages.SetTaskModeMapping) Task(org.apache.samza.rest.model.Task) LoggerFactory(org.slf4j.LoggerFactory) JobConfig(org.apache.samza.config.JobConfig) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) TaskAssignmentManager(org.apache.samza.container.grouper.task.TaskAssignmentManager) JobInstance(org.apache.samza.rest.proxy.job.JobInstance) URI(java.net.URI) MapConfig(org.apache.samza.config.MapConfig) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) StorageConfig(org.apache.samza.config.StorageConfig) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) InstallationFinder(org.apache.samza.rest.proxy.installation.InstallationFinder) ConfigFactory(org.apache.samza.config.ConfigFactory) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) SamzaException(org.apache.samza.SamzaException) LocalityManager(org.apache.samza.container.LocalityManager) CoordinatorStreamSystemConsumer(org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer) List(java.util.List) ReflectionUtil(org.apache.samza.util.ReflectionUtil) ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) Optional(java.util.Optional) InstallationRecord(org.apache.samza.rest.proxy.installation.InstallationRecord) Preconditions(com.google.common.base.Preconditions) Config(org.apache.samza.config.Config) SetTaskContainerMapping(org.apache.samza.coordinator.stream.messages.SetTaskContainerMapping) CoordinatorStreamUtil(org.apache.samza.util.CoordinatorStreamUtil) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Task(org.apache.samza.rest.model.Task) StorageConfig(org.apache.samza.config.StorageConfig) ArrayList(java.util.ArrayList) ProcessorLocality(org.apache.samza.job.model.ProcessorLocality) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) TaskAssignmentManager(org.apache.samza.container.grouper.task.TaskAssignmentManager) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) LocalityManager(org.apache.samza.container.LocalityManager)

Example 69 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestApplicationMasterRestClient method testGetMetricsSuccess.

@Test
public void testGetMetricsSuccess() throws IOException {
    SamzaApplicationState samzaAppState = createSamzaApplicationState();
    MetricsRegistryMap registry = new MetricsRegistryMap();
    assignMetricValues(samzaAppState, registry);
    String response = ApplicationMasterRestServlet.getMetrics(jsonMapper, registry);
    setupMockClientResponse(HttpStatus.SC_OK, "Success", response);
    ApplicationMasterRestClient client = new ApplicationMasterRestClient(mockClient, AM_HOST_NAME, AM_RPC_PORT);
    Map<String, Map<String, Object>> metricsResult = client.getMetrics();
    String group = SamzaAppMasterMetrics.class.getCanonicalName();
    assertEquals(1, metricsResult.size());
    assertTrue(metricsResult.containsKey(group));
    Map<String, Object> amMetricsGroup = metricsResult.get(group);
    assertEquals(8, amMetricsGroup.size());
    assertEquals(samzaAppState.runningProcessors.size(), amMetricsGroup.get("running-containers"));
    assertEquals(samzaAppState.neededProcessors.get(), amMetricsGroup.get("needed-containers"));
    assertEquals(samzaAppState.completedProcessors.get(), amMetricsGroup.get("completed-containers"));
    assertEquals(samzaAppState.failedContainers.get(), amMetricsGroup.get("failed-containers"));
    assertEquals(samzaAppState.releasedContainers.get(), amMetricsGroup.get("released-containers"));
    assertEquals(samzaAppState.processorCount.get(), amMetricsGroup.get("container-count"));
    assertEquals(samzaAppState.jobHealthy.get() ? 1 : 0, amMetricsGroup.get("job-healthy"));
    assertEquals(0, amMetricsGroup.get("container-from-previous-attempt"));
}
Also used : SamzaApplicationState(org.apache.samza.clustermanager.SamzaApplicationState) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Test(org.junit.Test)

Aggregations

MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)69 Test (org.junit.Test)27 MapConfig (org.apache.samza.config.MapConfig)24 Config (org.apache.samza.config.Config)23 HashMap (java.util.HashMap)17 JobConfig (org.apache.samza.config.JobConfig)12 Before (org.junit.Before)12 File (java.io.File)10 StorageConfig (org.apache.samza.config.StorageConfig)10 CoordinatorStreamStore (org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore)9 SamzaException (org.apache.samza.SamzaException)8 TaskName (org.apache.samza.container.TaskName)8 TaskModel (org.apache.samza.job.model.TaskModel)8 ClusterManagerConfig (org.apache.samza.config.ClusterManagerConfig)7 LocalityManager (org.apache.samza.container.LocalityManager)7 TaskContext (org.apache.samza.task.TaskContext)7 Partition (org.apache.samza.Partition)6 SystemStream (org.apache.samza.system.SystemStream)6 Map (java.util.Map)5 MetricsConfig (org.apache.samza.config.MetricsConfig)5