Examples with StreamPartitionCountMonitor org.apache.samza.coordinator.StreamPartitionCountMonitor used on opensource projects

Example 6 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testNoExistingJobModel.

@Test
public void testNoExistingJobModel() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}

Example 7 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testStopAfterStart.

@Test
public void testStopAfterStart() throws InterruptedException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    setUpDiagnosticsManager(jobModel);
    metadataResourceUtil(jobModel);
    // call start in order to set up monitors
    this.staticResourceJobCoordinator.start();
    // call stop to check that the expected components get shut down
    this.staticResourceJobCoordinator.stop();
    verify(this.jobCoordinatorListener).onJobModelExpired();
    verify(this.diagnosticsManager).stop();
    verify(streamPartitionCountMonitor).stop();
    verify(streamRegexMonitor).stop();
    verify(this.coordinatorCommunication).stop();
    verify(this.startpointManager).stop();
    verify(this.systemAdmins).stop();
    verify(this.jobCoordinatorListener).onCoordinatorStop();
}

Example 8 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testStartMissingOptionalComponents.

/**
 * Missing {@link StartpointManager}, {@link JobCoordinatorListener}, {@link StreamRegexMonitor}
 */
@Test
public void testStartMissingOptionalComponents() throws IOException {
    this.staticResourceJobCoordinator = spy(new StaticResourceJobCoordinator(PROCESSOR_ID, this.jobModelHelper, this.jobModelServingContext, this.coordinatorCommunication, this.jobCoordinatorMetadataManager, this.streamPartitionCountMonitorFactory, this.streamRegexMonitorFactory, Optional.empty(), this.changelogStreamManager, this.jobRestartSignal, this.metrics, this.systemAdmins, Optional.empty(), Optional.empty(), this.config));
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    when(this.streamRegexMonitorFactory.build(any(), any(), any())).thenReturn(Optional.empty());
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    doReturn(Optional.empty()).when(this.staticResourceJobCoordinator).buildDiagnosticsManager(JOB_NAME, JOB_ID, jobModel, CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, Optional.empty(), Optional.empty(), this.config);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verify(this.systemAdmins).start();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.staticResourceJobCoordinator).buildDiagnosticsManager(JOB_NAME, JOB_ID, jobModel, CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, Optional.empty(), Optional.empty(), this.config);
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, null, newMetadata, null);
    verifyZeroInteractions(this.jobCoordinatorListener, this.startpointManager);
}

Example 9 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class StaticResourceJobCoordinator method jobModelMonitors.

/*
   * Possible race condition: The partition count monitor queries for stream metadata when it is created, so if the
   * partition counts changed between the job model calculation and the creation of the partition count monitor, then
   * the monitor will not trigger an update to the job model. This method should be called right after calculating the
   * job model, in order to reduce the possible time in which a partition count change is missed. This issue also
   * exists in the older ClusterBasedJobCoordinator.
   * TODO This wouldn't be a problem if the partition count monitor used the job model to calculate initial metadata
   */
private JobModelMonitors jobModelMonitors(JobModel jobModel) {
    StreamPartitionCountMonitor streamPartitionCountMonitor = this.streamPartitionCountMonitorFactory.build(jobModel.getConfig(), streamsChanged -> this.jobRestartSignal.restartJob());
    Optional<StreamRegexMonitor> streamRegexMonitor = this.streamRegexMonitorFactory.build(jobModel, jobModel.getConfig(), (initialInputSet, newInputStreams, regexesMonitored) -> this.jobRestartSignal.restartJob());
    return new JobModelMonitors(streamPartitionCountMonitor, streamRegexMonitor.orElse(null));
}

Example 10 with StreamPartitionCountMonitor

use of org.apache.samza.coordinator.StreamPartitionCountMonitor in project samza by apache.

the class TestZkJobCoordinator method testShouldStopPartitionCountMonitorWhenStoppingTheJobCoordinator.

@Test
public void testShouldStopPartitionCountMonitorWhenStoppingTheJobCoordinator() {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(new JobModel(new MapConfig(), new HashMap<>()));
    ScheduleAfterDebounceTime mockDebounceTimer = Mockito.mock(ScheduleAfterDebounceTime.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, new MapConfig(), new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    StreamPartitionCountMonitor monitor = Mockito.mock(StreamPartitionCountMonitor.class);
    zkJobCoordinator.debounceTimer = mockDebounceTimer;
    zkJobCoordinator.streamPartitionCountMonitor = monitor;
    zkJobCoordinator.stop();
    Mockito.verify(monitor).stop();
}