Examples with StreamPartitionCountMonitor org.apache.samza.coordinator.StreamPartitionCountMonitor used on opensource projects

Example 1 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestClusterBasedJobCoordinator method testPartitionCountMonitorWithDurableStates.

@Test
public void testPartitionCountMonitorWithDurableStates() {
    configMap.put("stores.mystore.changelog", "mychangelog");
    configMap.put(JobConfig.JOB_CONTAINER_COUNT, "1");
    when(CoordinatorStreamUtil.readConfigFromCoordinatorStream(anyObject())).thenReturn(new MapConfig(configMap));
    Config config = new MapConfig(configMap);
    // mimic job runner code to write the config to coordinator stream
    CoordinatorStreamSystemProducer producer = new CoordinatorStreamSystemProducer(config, mock(MetricsRegistry.class));
    producer.writeConfig("test-job", config);
    ClusterBasedJobCoordinator clusterCoordinator = ClusterBasedJobCoordinatorRunner.createFromMetadataStore(config);
    // change the input system stream metadata
    MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "topic1", new Partition(1)), new ArrayList<>());
    StreamPartitionCountMonitor monitor = clusterCoordinator.getPartitionMonitor();
    monitor.updatePartitionCountMetric();
    assertEquals(clusterCoordinator.getAppStatus(), SamzaApplicationState.SamzaAppStatus.FAILED);
}

Example 2 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestClusterBasedJobCoordinator method testPartitionCountMonitorWithoutDurableStates.

@Test
public void testPartitionCountMonitorWithoutDurableStates() {
    configMap.put(JobConfig.JOB_CONTAINER_COUNT, "1");
    when(CoordinatorStreamUtil.readConfigFromCoordinatorStream(anyObject())).thenReturn(new MapConfig(configMap));
    Config config = new MapConfig(configMap);
    // mimic job runner code to write the config to coordinator stream
    CoordinatorStreamSystemProducer producer = new CoordinatorStreamSystemProducer(config, mock(MetricsRegistry.class));
    producer.writeConfig("test-job", config);
    ClusterBasedJobCoordinator clusterCoordinator = ClusterBasedJobCoordinatorRunner.createFromMetadataStore(config);
    // change the input system stream metadata
    MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "topic1", new Partition(1)), new ArrayList<>());
    StreamPartitionCountMonitor monitor = clusterCoordinator.getPartitionMonitor();
    monitor.updatePartitionCountMetric();
    assertEquals(clusterCoordinator.getAppStatus(), SamzaApplicationState.SamzaAppStatus.UNDEFINED);
}

Example 3 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestZkJobCoordinator method testShouldStartPartitionCountMonitorOnBecomingLeader.

@Test
public void testShouldStartPartitionCountMonitorOnBecomingLeader() {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(new JobModel(new MapConfig(), new HashMap<>()));
    ScheduleAfterDebounceTime mockDebounceTimer = Mockito.mock(ScheduleAfterDebounceTime.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, new MapConfig(), new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    StreamPartitionCountMonitor monitor = Mockito.mock(StreamPartitionCountMonitor.class);
    zkJobCoordinator.debounceTimer = mockDebounceTimer;
    zkJobCoordinator.streamPartitionCountMonitor = monitor;
    doReturn(monitor).when(zkJobCoordinator).getPartitionCountMonitor();
    ZkJobCoordinator.LeaderElectorListenerImpl listener = zkJobCoordinator.new LeaderElectorListenerImpl();
    listener.onBecomingLeader();
    Mockito.verify(monitor).start();
}

Example 4 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testSameJobModelAsPrevious.

@Test
public void testSameJobModelAsPrevious() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, null, null);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}

Example 5 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method setupStreamPartitionCountMonitor.

/**
 * Set up {@link StreamPartitionCountMonitorFactory} to return a mock {@link StreamPartitionCountMonitor}.
 */
private StreamPartitionCountMonitor setupStreamPartitionCountMonitor(Config config) {
    StreamPartitionCountMonitor streamPartitionCountMonitor = mock(StreamPartitionCountMonitor.class);
    when(this.streamPartitionCountMonitorFactory.build(eq(config), any())).thenReturn(streamPartitionCountMonitor);
    return streamPartitionCountMonitor;
}