Examples with MetricsRegistry org.apache.samza.metrics.MetricsRegistry used on opensource projects

Example 11 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class TestKinesisSystemFactory method testGetConsumer.

@Test
public void testGetConsumer() {
    String systemName = "test";
    Config config = buildKinesisConsumerConfig(systemName);
    KinesisSystemFactory factory = new KinesisSystemFactory();
    MetricsRegistry metricsRegistry = new NoOpMetricsRegistry();
    Assert.assertNotSame(factory.getConsumer("test", config, metricsRegistry), factory.getAdmin(systemName, config));
}

Example 12 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class ContainerStorageManager method createTaskStores.

/**
 * Create taskStores for all stores in storesToCreate.
 * The store mode is chosen as read-write mode.
 */
private Map<TaskName, Map<String, StorageEngine>> createTaskStores(Set<String> storesToCreate, ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext, Map<String, StorageEngineFactory<Object, Object>> storageEngineFactories, Map<String, Serde<Object>> serdes, Map<TaskName, TaskInstanceMetrics> taskInstanceMetrics, Map<TaskName, TaskInstanceCollector> taskInstanceCollectors) {
    Map<TaskName, Map<String, StorageEngine>> taskStores = new HashMap<>();
    StorageConfig storageConfig = new StorageConfig(config);
    // iterate over each task and each storeName
    for (Map.Entry<TaskName, TaskModel> task : containerModel.getTasks().entrySet()) {
        TaskName taskName = task.getKey();
        TaskModel taskModel = task.getValue();
        if (!taskStores.containsKey(taskName)) {
            taskStores.put(taskName, new HashMap<>());
        }
        for (String storeName : storesToCreate) {
            List<String> storeBackupManagers = storageConfig.getStoreBackupFactories(storeName);
            // A store is considered durable if it is backed by a changelog or another backupManager factory
            boolean isDurable = changelogSystemStreams.containsKey(storeName) || !storeBackupManagers.isEmpty();
            boolean isSideInput = this.sideInputStoreNames.contains(storeName);
            // Use the logged-store-base-directory for change logged stores and sideInput stores, and non-logged-store-base-dir
            // for non logged stores
            File storeBaseDir = isDurable || isSideInput ? this.loggedStoreBaseDirectory : this.nonLoggedStoreBaseDirectory;
            File storeDirectory = storageManagerUtil.getTaskStoreDir(storeBaseDir, storeName, taskName, taskModel.getTaskMode());
            this.storeDirectoryPaths.add(storeDirectory.toPath());
            // if taskInstanceMetrics are specified use those for store metrics,
            // otherwise (in case of StorageRecovery) use a blank MetricsRegistryMap
            MetricsRegistry storeMetricsRegistry = taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap();
            StorageEngine storageEngine = createStore(storeName, storeDirectory, taskModel, jobContext, containerContext, storageEngineFactories, serdes, storeMetricsRegistry, taskInstanceCollectors.get(taskName), StorageEngineFactory.StoreMode.ReadWrite, this.changelogSystemStreams, this.config);
            // add created store to map
            taskStores.get(taskName).put(storeName, storageEngine);
            LOG.info("Created task store {} in read-write mode for task {} in path {}", storeName, taskName, storeDirectory.getAbsolutePath());
        }
    }
    return taskStores;
}

Example 13 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class TestLocalTableWrite method setUp.

@Before
public void setUp() {
    putNs = new Timer("");
    putAllNs = new Timer("");
    deleteNs = new Timer("");
    deleteAllNs = new Timer("");
    flushNs = new Timer("");
    numPuts = new Counter("");
    numPutAlls = new Counter("");
    numDeletes = new Counter("");
    numDeleteAlls = new Counter("");
    numFlushes = new Counter("");
    putCallbackNs = new Timer("");
    deleteCallbackNs = new Timer("");
    metricsRegistry = mock(MetricsRegistry.class);
    String groupName = LocalTable.class.getSimpleName();
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-put-ns")).thenReturn(putNs);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-putAll-ns")).thenReturn(putAllNs);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-delete-ns")).thenReturn(deleteNs);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-deleteAll-ns")).thenReturn(deleteAllNs);
    when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-puts")).thenReturn(numPuts);
    when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-putAlls")).thenReturn(numPutAlls);
    when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-deletes")).thenReturn(numDeletes);
    when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-deleteAlls")).thenReturn(numDeleteAlls);
    when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-flushes")).thenReturn(numFlushes);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-put-callback-ns")).thenReturn(putCallbackNs);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-delete-callback-ns")).thenReturn(deleteCallbackNs);
    when(metricsRegistry.newTimer(groupName, TABLE_ID + "-flush-ns")).thenReturn(flushNs);
    kvStore = mock(KeyValueStore.class);
}

Example 14 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class TestRemoteTableEndToEnd method createMockContext.

private Context createMockContext() {
    MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
    doReturn(new Counter("")).when(metricsRegistry).newCounter(anyString(), anyString());
    doReturn(new Timer("")).when(metricsRegistry).newTimer(anyString(), anyString());
    Context context = new MockContext();
    doReturn(new MapConfig()).when(context.getJobContext()).getConfig();
    doReturn(metricsRegistry).when(context.getContainerContext()).getContainerMetricsRegistry();
    return context;
}

Example 15 with MetricsRegistry

use of org.apache.samza.metrics.MetricsRegistry in project samza by apache.

the class YarnJobValidationTool method validateJmxMetrics.

public void validateJmxMetrics() throws Exception {
    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(config, metricsRegistry);
    coordinatorStreamStore.init();
    try {
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE));
        validator.init(config);
        LocalityModel localityModel = localityManager.readLocality();
        for (ProcessorLocality processorLocality : localityModel.getProcessorLocalities().values()) {
            String containerId = processorLocality.id();
            String jmxUrl = processorLocality.jmxTunnelingUrl();
            if (StringUtils.isNotBlank(jmxUrl)) {
                log.info("validate container " + containerId + " metrics with JMX: " + jmxUrl);
                JmxMetricsAccessor jmxMetrics = new JmxMetricsAccessor(jmxUrl);
                jmxMetrics.connect();
                validator.validate(jmxMetrics);
                jmxMetrics.close();
                log.info("validate container " + containerId + " successfully");
            }
        }
        validator.complete();
    } finally {
        coordinatorStreamStore.close();
    }
}