Examples with MetricsRegistryMap org.apache.samza.metrics.MetricsRegistryMap used on opensource projects

Example 51 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestZkMetadataStore method beforeTest.

@Before
public void beforeTest() {
    String testZkConnectionString = String.format("%s:%s", LOCALHOST, zkServer.getPort());
    Config zkConfig = new MapConfig(ImmutableMap.of(ZkConfig.ZK_CONNECT, testZkConnectionString));
    zkMetadataStore = new ZkMetadataStoreFactory().getMetadataStore(String.format("%s", RandomStringUtils.randomAlphabetic(5)), zkConfig, new MetricsRegistryMap());
}

Example 52 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestMetricsSnapshotReporterFactory method testGetProducer.

@Test
public void testGetProducer() {
    Config config = new MapConfig(ImmutableMap.of("metrics.reporter.metrics-reporter.stream", "system0.stream0", "systems.system0.samza.factory", MockSystemFactory.class.getName()));
    assertEquals(SYSTEM_PRODUCER, this.factory.getProducer(REPORTER, config, new MetricsRegistryMap()));
}

Example 53 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestMetricsSnapshotReporter method testMetricsEmission.

@Test
public void testMetricsEmission() {
    // setup
    serializer = null;
    String source = "testSource";
    String group = "someGroup";
    String metricName = "someName";
    MetricsRegistryMap registry = new MetricsRegistryMap();
    metricsSnapshotReporter = getMetricsSnapshotReporter(Optional.empty());
    registry.newGauge(group, metricName, 42);
    metricsSnapshotReporter.register(source, registry);
    ArgumentCaptor<OutgoingMessageEnvelope> outgoingMessageEnvelopeArgumentCaptor = ArgumentCaptor.forClass(OutgoingMessageEnvelope.class);
    // run
    metricsSnapshotReporter.run();
    // assert
    verify(producer, times(1)).send(eq(source), outgoingMessageEnvelopeArgumentCaptor.capture());
    verify(producer, times(1)).flush(eq(source));
    List<OutgoingMessageEnvelope> envelopes = outgoingMessageEnvelopeArgumentCaptor.getAllValues();
    Assert.assertEquals(1, envelopes.size());
    MetricsSnapshot metricsSnapshot = (MetricsSnapshot) envelopes.get(0).getMessage();
    MetricsHeader expectedHeader = new MetricsHeader(JOB_NAME, JOB_ID, CONTAINER_NAME, "", Optional.of(""), source, TASK_VERSION, SAMZA_VERSION, HOSTNAME, SEND_TIME, RESET_TIME);
    Assert.assertEquals(expectedHeader, metricsSnapshot.getHeader());
    Map<String, Map<String, Object>> metricMap = metricsSnapshot.getMetrics().getAsMap();
    Assert.assertEquals(1, metricMap.size());
    Assert.assertTrue(metricMap.containsKey(group));
    Assert.assertTrue(metricMap.get(group).containsKey(metricName));
    Assert.assertEquals(42, metricMap.get(group).get(metricName));
}

Example 54 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class ContainerStorageManager method createTaskStores.

/**
 * Create taskStores for all stores in storesToCreate.
 * The store mode is chosen as read-write mode.
 */
private Map<TaskName, Map<String, StorageEngine>> createTaskStores(Set<String> storesToCreate, ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext, Map<String, StorageEngineFactory<Object, Object>> storageEngineFactories, Map<String, Serde<Object>> serdes, Map<TaskName, TaskInstanceMetrics> taskInstanceMetrics, Map<TaskName, TaskInstanceCollector> taskInstanceCollectors) {
    Map<TaskName, Map<String, StorageEngine>> taskStores = new HashMap<>();
    StorageConfig storageConfig = new StorageConfig(config);
    // iterate over each task and each storeName
    for (Map.Entry<TaskName, TaskModel> task : containerModel.getTasks().entrySet()) {
        TaskName taskName = task.getKey();
        TaskModel taskModel = task.getValue();
        if (!taskStores.containsKey(taskName)) {
            taskStores.put(taskName, new HashMap<>());
        }
        for (String storeName : storesToCreate) {
            List<String> storeBackupManagers = storageConfig.getStoreBackupFactories(storeName);
            // A store is considered durable if it is backed by a changelog or another backupManager factory
            boolean isDurable = changelogSystemStreams.containsKey(storeName) || !storeBackupManagers.isEmpty();
            boolean isSideInput = this.sideInputStoreNames.contains(storeName);
            // Use the logged-store-base-directory for change logged stores and sideInput stores, and non-logged-store-base-dir
            // for non logged stores
            File storeBaseDir = isDurable || isSideInput ? this.loggedStoreBaseDirectory : this.nonLoggedStoreBaseDirectory;
            File storeDirectory = storageManagerUtil.getTaskStoreDir(storeBaseDir, storeName, taskName, taskModel.getTaskMode());
            this.storeDirectoryPaths.add(storeDirectory.toPath());
            // if taskInstanceMetrics are specified use those for store metrics,
            // otherwise (in case of StorageRecovery) use a blank MetricsRegistryMap
            MetricsRegistry storeMetricsRegistry = taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap();
            StorageEngine storageEngine = createStore(storeName, storeDirectory, taskModel, jobContext, containerContext, storageEngineFactories, serdes, storeMetricsRegistry, taskInstanceCollectors.get(taskName), StorageEngineFactory.StoreMode.ReadWrite, this.changelogSystemStreams, this.config);
            // add created store to map
            taskStores.get(taskName).put(storeName, storageEngine);
            LOG.info("Created task store {} in read-write mode for task {} in path {}", storeName, taskName, storeDirectory.getAbsolutePath());
        }
    }
    return taskStores;
}

Example 55 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestContainerPlacementActions method testAlwaysMoveToAnyHostForHostAffinityDisabled.

@Test(timeout = 10000)
public void testAlwaysMoveToAnyHostForHostAffinityDisabled() throws Exception {
    Map<String, String> conf = new HashMap<>();
    conf.putAll(getConfigWithHostAffinityAndRetries(false, 1, true));
    SamzaApplicationState state = new SamzaApplicationState(JobModelManagerTestUtil.getJobModelManager(getConfig(), 2, this.server));
    ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);
    MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback, state);
    FaultDomainManager faultDomainManager = mock(FaultDomainManager.class);
    ContainerManager containerManager = new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, false, false, localityManager, faultDomainManager, config);
    MockContainerAllocatorWithoutHostAffinity allocatorWithoutHostAffinity = new MockContainerAllocatorWithoutHostAffinity(clusterResourceManager, new MapConfig(conf), state, containerManager);
    ContainerProcessManager cpm = new ContainerProcessManager(new ClusterManagerConfig(new MapConfig(getConfig(), getConfigWithHostAffinityAndRetries(false, 1, true))), state, new MetricsRegistryMap(), clusterResourceManager, Optional.of(allocatorWithoutHostAffinity), containerManager, localityManager, false);
    // Mimic Cluster Manager returning any request
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                List<SamzaResource> resources = (List<SamzaResource>) args[0];
                SamzaResource preferredResource = resources.get(0);
                SamzaResource anyResource = new SamzaResource(preferredResource.getNumCores(), preferredResource.getMemoryMb(), "host-" + RandomStringUtils.randomAlphanumeric(5), preferredResource.getContainerId());
                cpm.onResourcesAvailable(ImmutableList.of(anyResource));
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesAvailable(anyList());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onStreamProcessorLaunchSuccess((SamzaResource) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onStreamProcessorLaunchSuccess(any());
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            new Thread(() -> {
                Object[] args = invocation.getArguments();
                cpm.onResourcesCompleted((List<SamzaResourceStatus>) args[0]);
            }, "AMRMClientAsync").start();
            return null;
        }
    }).when(callback).onResourcesCompleted(anyList());
    cpm.start();
    // This spawns async start request and waits for async requests to complete
    if (!allocatorWithoutHostAffinity.awaitContainersStart(2, 3, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    while (state.runningProcessors.size() != 2) {
        Thread.sleep(100);
    }
    // App is in running state with two containers running
    assertEquals(state.runningProcessors.size(), 2);
    assertEquals(state.preferredHostRequests.get(), 0);
    assertEquals(state.anyHostRequests.get(), 2);
    String previousHostOfContainer1 = state.runningProcessors.get("0").getHost();
    String previousHostOfContainer2 = state.runningProcessors.get("1").getHost();
    // Initiate container placement action to move a container with container id 0
    ContainerPlacementRequestMessage requestMessage = new ContainerPlacementRequestMessage(UUID.randomUUID(), "app-attempt-001", "0", "host-3", System.currentTimeMillis());
    ContainerPlacementMetadata metadata = containerManager.registerContainerPlacementActionForTest(requestMessage, allocatorWithoutHostAffinity);
    // Wait for the ControlAction to complete and spawn an async request
    if (!allocatorWithoutHostAffinity.awaitContainersStart(1, 3, TimeUnit.SECONDS)) {
        fail("timed out waiting for the containers to start");
    }
    Optional<ContainerPlacementResponseMessage> responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    while (true) {
        if (metadata.getActionStatus() == ContainerPlacementMessage.StatusCode.SUCCEEDED && responseMessage.isPresent() && responseMessage.get().getStatusCode() == ContainerPlacementMessage.StatusCode.SUCCEEDED) {
            break;
        }
        Thread.sleep(100);
        responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    }
    // We should have no preferred host request
    assertEquals(0, state.preferredHostRequests.get());
    // We should have one more ANY_HOST request
    assertEquals(3, state.anyHostRequests.get());
    assertEquals(2, state.runningProcessors.size());
    assertNotEquals(previousHostOfContainer1, state.runningProcessors.get("0").getHost());
    // Container 2 should not be affected
    assertEquals(previousHostOfContainer2, state.runningProcessors.get("1").getHost());
    assertEquals(3, state.anyHostRequests.get());
    // Action should success
    assertEquals(ContainerPlacementMessage.StatusCode.SUCCEEDED, metadata.getActionStatus());
    assertTrue(responseMessage.isPresent());
    assertEquals(responseMessage.get().getStatusCode(), ContainerPlacementMessage.StatusCode.SUCCEEDED);
    assertResponseMessage(responseMessage.get(), requestMessage);
    /**
     * Inject a duplicate request and check it is not accepted
     */
    ContainerPlacementRequestMessage duplicateRequestToBeIgnored = new ContainerPlacementRequestMessage(requestMessage.getUuid(), "app-attempt-001", "1", "host-3", System.currentTimeMillis());
    // Request with a dup uuid should not be accepted
    metadata = containerManager.registerContainerPlacementActionForTest(duplicateRequestToBeIgnored, allocatorWithoutHostAffinity);
    // metadata should be from the previous completed action
    assertTrue(metadata == null || metadata.getUuid() != duplicateRequestToBeIgnored.getUuid());
    responseMessage = containerPlacementMetadataStore.readContainerPlacementResponseMessage(requestMessage.getUuid());
    assertTrue(responseMessage.isPresent());
    assertEquals(responseMessage.get().getStatusCode(), ContainerPlacementMessage.StatusCode.BAD_REQUEST);
    assertResponseMessage(responseMessage.get(), duplicateRequestToBeIgnored);
    // Request shall be deleted as soon as it is acted upon
    assertFalse(containerPlacementMetadataStore.readContainerPlacementRequestMessage(requestMessage.getUuid()).isPresent());
}