Examples with MetricsRegistry org.apache.samza.metrics.MetricsRegistry used on opensource projects

Example 1 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class OperatorImpl method init.

/**
   * Initialize this {@link OperatorImpl} and its user-defined functions.
   *
   * @param config  the {@link Config} for the task
   * @param context  the {@link TaskContext} for the task
   */
public final void init(Config config, TaskContext context) {
    String opName = getOperatorSpec().getOpName();
    if (initialized) {
        throw new IllegalStateException(String.format("Attempted to initialize Operator %s more than once.", opName));
    }
    if (closed) {
        throw new IllegalStateException(String.format("Attempted to initialize Operator %s after it was closed.", opName));
    }
    this.highResClock = createHighResClock(config);
    registeredOperators = new HashSet<>();
    MetricsRegistry metricsRegistry = context.getMetricsRegistry();
    this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opName + "-messages");
    this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opName + "-handle-message-ns");
    this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opName + "-handle-timer-ns");
    handleInit(config, context);
    initialized = true;
}

Example 2 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class TestBlobStoreBackupManager method setup.

@Before
public void setup() throws Exception {
    when(clock.currentTimeMillis()).thenReturn(1234567L);
    // setup test local and remote snapshots
    indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
    // setup test store name and SCMs map
    testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair);
    // setup: setup task backup manager with expected storeName->storageEngine map
    testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null));
    mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId)));
    Config config = new MapConfig(mapConfig);
    // Mock - return snapshot index for blob id from test blob store map
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))).then((Answer<CompletableFuture<SnapshotIndex>>) invocation -> {
        String blobId = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture(testBlobStore.get(blobId));
    });
    // doNothing().when(blobStoreManager).init();
    when(taskModel.getTaskName().getTaskName()).thenReturn(taskName);
    when(taskModel.getTaskMode()).thenReturn(TaskMode.Active);
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
    when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge);
    when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge);
    when(atomicLongGauge.getValue()).thenReturn(new AtomicLong());
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer);
    blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
    blobStoreBackupManager = new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, blobStoreTaskBackupMetrics, config, Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager);
}

Example 3 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class ContainerStorageManager method createTaskRestoreManagers.

private Map<String, TaskRestoreManager> createTaskRestoreManagers(Map<String, StateBackendFactory> factories, Map<String, Set<String>> backendFactoryStoreNames, Clock clock, SamzaContainerMetrics samzaContainerMetrics, TaskName taskName, TaskModel taskModel) {
    // Get the factories for the task based on the stores of the tasks to be restored from the factory
    // backendFactoryName -> restoreManager
    Map<String, TaskRestoreManager> backendFactoryRestoreManagers = new HashMap<>();
    MetricsRegistry taskMetricsRegistry = taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap();
    backendFactoryStoreNames.forEach((factoryName, storeNames) -> {
        StateBackendFactory factory = factories.get(factoryName);
        KafkaChangelogRestoreParams kafkaChangelogRestoreParams = new KafkaChangelogRestoreParams(storeConsumers, inMemoryStores.get(taskName), systemAdmins.getSystemAdmins(), storageEngineFactories, serdes, taskInstanceCollectors.get(taskName));
        TaskRestoreManager restoreManager = factory.getRestoreManager(jobContext, containerContext, taskModel, restoreExecutor, taskMetricsRegistry, storeNames, config, clock, loggedStoreBaseDirectory, nonLoggedStoreBaseDirectory, kafkaChangelogRestoreParams);
        backendFactoryRestoreManagers.put(factoryName, restoreManager);
    });
    samzaContainerMetrics.addStoresRestorationGauge(taskName);
    return backendFactoryRestoreManagers;
}

Example 4 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class KafkaChangelogStateBackendFactory method getRestoreManager.

@Override
public TaskRestoreManager getRestoreManager(JobContext jobContext, ContainerContext containerContext, TaskModel taskModel, ExecutorService restoreExecutor, MetricsRegistry metricsRegistry, Set<String> storesToRestore, Config config, Clock clock, File loggedStoreBaseDir, File nonLoggedStoreBaseDir, KafkaChangelogRestoreParams kafkaChangelogRestoreParams) {
    Map<String, SystemStream> storeChangelogs = new StorageConfig(config).getStoreChangelogs();
    Set<SystemStreamPartition> changelogSSPs = storeChangelogs.values().stream().flatMap(ss -> containerContext.getContainerModel().getTasks().values().stream().map(tm -> new SystemStreamPartition(ss, tm.getChangelogPartition()))).collect(Collectors.toSet());
    // filter out standby store-ssp pairs
    Map<String, SystemStream> filteredStoreChangelogs = filterStandbySystemStreams(storeChangelogs, containerContext.getContainerModel());
    SystemAdmins systemAdmins = new SystemAdmins(kafkaChangelogRestoreParams.getSystemAdmins());
    if (new TaskConfig(config).getTransactionalStateRestoreEnabled()) {
        return new TransactionalStateTaskRestoreManager(storesToRestore, jobContext, containerContext, taskModel, restoreExecutor, filteredStoreChangelogs, kafkaChangelogRestoreParams.getInMemoryStores(), kafkaChangelogRestoreParams.getStorageEngineFactories(), kafkaChangelogRestoreParams.getSerdes(), systemAdmins, kafkaChangelogRestoreParams.getStoreConsumers(), metricsRegistry, kafkaChangelogRestoreParams.getCollector(), getSspCache(systemAdmins, clock, changelogSSPs), loggedStoreBaseDir, nonLoggedStoreBaseDir, config, clock);
    } else {
        return new NonTransactionalStateTaskRestoreManager(storesToRestore, jobContext, containerContext, taskModel, restoreExecutor, filteredStoreChangelogs, kafkaChangelogRestoreParams.getInMemoryStores(), kafkaChangelogRestoreParams.getStorageEngineFactories(), kafkaChangelogRestoreParams.getSerdes(), systemAdmins, getStreamCache(systemAdmins, clock), kafkaChangelogRestoreParams.getStoreConsumers(), metricsRegistry, kafkaChangelogRestoreParams.getCollector(), jobContext.getJobModel().getMaxChangeLogStreamPartitions(), loggedStoreBaseDir, nonLoggedStoreBaseDir, config, clock);
    }
}

Example 5 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class TestRemoteTableDescriptor method createMockContext.

private Context createMockContext(TableDescriptor tableDescriptor) {
    Context context = mock(Context.class);
    ContainerContext containerContext = mock(ContainerContext.class);
    when(context.getContainerContext()).thenReturn(containerContext);
    MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(mock(Timer.class));
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(mock(Counter.class));
    when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry);
    TaskContextImpl taskContext = mock(TaskContextImpl.class);
    when(context.getTaskContext()).thenReturn(taskContext);
    TaskName taskName = new TaskName("MyTask");
    TaskModel taskModel = mock(TaskModel.class);
    when(taskModel.getTaskName()).thenReturn(taskName);
    when(context.getTaskContext().getTaskModel()).thenReturn(taskModel);
    ContainerModel containerModel = mock(ContainerModel.class);
    when(containerModel.getTasks()).thenReturn(ImmutableMap.of(taskName, taskModel));
    when(containerContext.getContainerModel()).thenReturn(containerModel);
    String containerId = "container-1";
    JobModel jobModel = mock(JobModel.class);
    when(taskContext.getJobModel()).thenReturn(jobModel);
    when(jobModel.getContainers()).thenReturn(ImmutableMap.of(containerId, containerModel));
    JobContext jobContext = mock(JobContext.class);
    Config jobConfig = new MapConfig(tableDescriptor.toConfig(new MapConfig()));
    when(jobContext.getConfig()).thenReturn(jobConfig);
    when(context.getJobContext()).thenReturn(jobContext);
    return context;
}