Examples with MetricsRegistryMap org.apache.samza.metrics.MetricsRegistryMap used on opensource projects

Example 46 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class StorageRecovery method getTaskStorageManagers.

/**
   * create one TaskStorageManager for each task. Add all of them to the
   * List<TaskStorageManager>
   */
@SuppressWarnings({ "unchecked", "rawtypes" })
private void getTaskStorageManagers() {
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(Util.javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
    for (ContainerModel containerModel : containers.values()) {
        HashMap<String, StorageEngine> taskStores = new HashMap<String, StorageEngine>();
        SamzaContainerContext containerContext = new SamzaContainerContext(containerModel.getProcessorId(), jobConfig, containerModel.getTasks().keySet());
        for (TaskModel taskModel : containerModel.getTasks().values()) {
            HashMap<String, SystemConsumer> storeConsumers = getStoreConsumers();
            for (Entry<String, StorageEngineFactory<?, ?>> entry : storageEngineFactories.entrySet()) {
                String storeName = entry.getKey();
                if (changeLogSystemStreams.containsKey(storeName)) {
                    SystemStreamPartition changeLogSystemStreamPartition = new SystemStreamPartition(changeLogSystemStreams.get(storeName), taskModel.getChangelogPartition());
                    File storePartitionDir = TaskStorageManager.getStorePartitionDir(storeBaseDir, storeName, taskModel.getTaskName());
                    log.info("Got storage engine directory: " + storePartitionDir);
                    StorageEngine storageEngine = (entry.getValue()).getStorageEngine(storeName, storePartitionDir, (Serde) new ByteSerde(), (Serde) new ByteSerde(), null, new MetricsRegistryMap(), changeLogSystemStreamPartition, containerContext);
                    taskStores.put(storeName, storageEngine);
                }
            }
            TaskStorageManager taskStorageManager = new TaskStorageManager(taskModel.getTaskName(), Util.javaMapAsScalaMap(taskStores), Util.javaMapAsScalaMap(storeConsumers), Util.javaMapAsScalaMap(changeLogSystemStreams), maxPartitionNumber, streamMetadataCache, storeBaseDir, storeBaseDir, taskModel.getChangelogPartition(), Util.javaMapAsScalaMap(systemAdmins), new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
            taskStorageManagers.add(taskStorageManager);
        }
    }
}

Example 47 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class StorageRecovery method getContainerModels.

/**
 * Build ContainerModels from job config file and put the results in the containerModels map.
 */
private void getContainerModels() {
    MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(jobConfig, metricsRegistryMap);
    coordinatorStreamStore.init();
    try {
        Config configFromCoordinatorStream = CoordinatorStreamUtil.readConfigFromCoordinatorStream(coordinatorStreamStore);
        ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(coordinatorStreamStore);
        JobModelManager jobModelManager = JobModelManager.apply(configFromCoordinatorStream, changelogStreamManager.readPartitionMapping(), coordinatorStreamStore, metricsRegistryMap);
        JobModel jobModel = jobModelManager.jobModel();
        this.jobModel = jobModel;
        containers = jobModel.getContainers();
    } finally {
        coordinatorStreamStore.close();
    }
}

Example 48 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class StorageRecovery method getContainerStorageManagers.

/**
 * create one TaskStorageManager for each task. Add all of them to the
 * List<TaskStorageManager>
 */
@SuppressWarnings("rawtypes")
private void getContainerStorageManagers() {
    Set<String> factoryClasses = new StorageConfig(jobConfig).getRestoreFactories();
    Map<String, StateBackendFactory> stateBackendFactories = factoryClasses.stream().collect(Collectors.toMap(factoryClass -> factoryClass, factoryClass -> ReflectionUtil.getObj(factoryClass, StateBackendFactory.class)));
    Clock clock = SystemClock.instance();
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock);
    // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways
    Map<String, SystemFactory> systemFactories = new SystemConfig(jobConfig).getSystemFactories();
    CheckpointManager checkpointManager = new TaskConfig(jobConfig).getCheckpointManager(new MetricsRegistryMap()).orElse(null);
    for (ContainerModel containerModel : containers.values()) {
        ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap());
        ContainerStorageManager containerStorageManager = new ContainerStorageManager(checkpointManager, containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams, new HashMap<>(), storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(), new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap(), ""), JobContextImpl.fromConfigWithDefaults(jobConfig, jobModel), containerContext, stateBackendFactories, new HashMap<>(), storeBaseDir, storeBaseDir, null, new SystemClock());
        this.containerStorageManagers.put(containerModel.getId(), containerStorageManager);
    }
}

Example 49 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class StreamProcessor method createSamzaContainer.

@VisibleForTesting
SamzaContainer createSamzaContainer(String processorId, JobModel jobModel) {
    // Creating diagnostics manager and wiring it respectively
    String jobName = new JobConfig(config).getName().get();
    String jobId = new JobConfig(config).getJobId();
    Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(jobName, jobId, jobModel, processorId, Optional.empty(), Optional.empty(), config);
    // Metadata store lifecycle managed outside of the SamzaContainer.
    // All manager lifecycles are managed in the SamzaContainer including startpointManager
    StartpointManager startpointManager = null;
    if (metadataStore != null && new JobConfig(config).getStartpointEnabled()) {
        startpointManager = new StartpointManager(metadataStore);
    } else if (!new JobConfig(config).getStartpointEnabled()) {
        LOGGER.warn("StartpointManager not instantiated because startpoints is not enabled");
    } else {
        LOGGER.warn("StartpointManager cannot be instantiated because no metadata store defined for this stream processor");
    }
    /*
     * StreamProcessor has a metricsRegistry instance variable, but StreamProcessor registers its metrics on its own
     * with the reporters. Therefore, don't reuse the StreamProcessor.metricsRegistry, because SamzaContainer also
     * registers the registry, and that will result in unnecessary duplicate metrics.
     */
    MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
    return SamzaContainer.apply(processorId, jobModel, ScalaJavaUtil.toScalaMap(this.customMetricsReporter), metricsRegistryMap, this.taskFactory, JobContextImpl.fromConfigWithDefaults(this.config, jobModel), Option.apply(this.applicationDefinedContainerContextFactoryOptional.orElse(null)), Option.apply(this.applicationDefinedTaskContextFactoryOptional.orElse(null)), Option.apply(this.externalContextOptional.orElse(null)), null, startpointManager, Option.apply(diagnosticsManager.orElse(null)));
}

Example 50 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestZkJobCoordinator method testZookeeperSessionMetricsAreUpdatedCorrectly.

@Test
public void testZookeeperSessionMetricsAreUpdatedCorrectly() {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(new JobModel(new MapConfig(), new HashMap<>()));
    ScheduleAfterDebounceTime mockDebounceTimer = Mockito.mock(ScheduleAfterDebounceTime.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, new MapConfig(), new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    zkJobCoordinator.debounceTimer = mockDebounceTimer;
    zkJobCoordinator.zkSessionMetrics = new ZkSessionMetrics(new MetricsRegistryMap());
    final ZkSessionStateChangedListener zkSessionStateChangedListener = zkJobCoordinator.new ZkSessionStateChangedListener();
    zkSessionStateChangedListener.handleStateChanged(Watcher.Event.KeeperState.Disconnected);
    zkSessionStateChangedListener.handleStateChanged(Watcher.Event.KeeperState.SyncConnected);
    zkSessionStateChangedListener.handleStateChanged(Watcher.Event.KeeperState.AuthFailed);
    Assert.assertEquals(1, zkJobCoordinator.zkSessionMetrics.zkSessionErrors.getCount());
    zkSessionStateChangedListener.handleSessionEstablishmentError(new SamzaException("Test exception"));
    Assert.assertEquals(1, zkJobCoordinator.zkSessionMetrics.zkSessionDisconnects.getCount());
    Assert.assertEquals(1, zkJobCoordinator.zkSessionMetrics.zkSyncConnected.getCount());
    Assert.assertEquals(2, zkJobCoordinator.zkSessionMetrics.zkSessionErrors.getCount());
}