Examples with JobCoordinatorMetadata org.apache.samza.job.JobCoordinatorMetadata used on opensource projects

Example 1 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class ClusterBasedJobCoordinator method generateAndUpdateJobCoordinatorMetadata.

/**
 * Generate the job coordinator metadata for current application attempt and checks for changes in the
 * metadata from the previous attempt and writes the updates metadata to coordinator stream.
 *
 * @param jobModel job model used to generate the job coordinator metadata
 */
@VisibleForTesting
void generateAndUpdateJobCoordinatorMetadata(JobModel jobModel) {
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = createJobCoordinatorMetadataManager();
    JobCoordinatorMetadata previousMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    JobCoordinatorMetadata newMetadata = jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, config);
    if (!jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata).isEmpty()) {
        jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(newMetadata);
        metadataChangedAcrossAttempts = true;
    }
}

Example 2 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestStaticResourceJobCoordinator method testNoExistingJobModel.

@Test
public void testNoExistingJobModel() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}

Example 3 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestStaticResourceJobCoordinator method testStartMissingOptionalComponents.

/**
 * Missing {@link StartpointManager}, {@link JobCoordinatorListener}, {@link StreamRegexMonitor}
 */
@Test
public void testStartMissingOptionalComponents() throws IOException {
    this.staticResourceJobCoordinator = spy(new StaticResourceJobCoordinator(PROCESSOR_ID, this.jobModelHelper, this.jobModelServingContext, this.coordinatorCommunication, this.jobCoordinatorMetadataManager, this.streamPartitionCountMonitorFactory, this.streamRegexMonitorFactory, Optional.empty(), this.changelogStreamManager, this.jobRestartSignal, this.metrics, this.systemAdmins, Optional.empty(), Optional.empty(), this.config));
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    when(this.streamRegexMonitorFactory.build(any(), any(), any())).thenReturn(Optional.empty());
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    doReturn(Optional.empty()).when(this.staticResourceJobCoordinator).buildDiagnosticsManager(JOB_NAME, JOB_ID, jobModel, CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, Optional.empty(), Optional.empty(), this.config);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verify(this.systemAdmins).start();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.staticResourceJobCoordinator).buildDiagnosticsManager(JOB_NAME, JOB_ID, jobModel, CoordinationConstants.JOB_COORDINATOR_CONTAINER_NAME, Optional.empty(), Optional.empty(), this.config);
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, null, newMetadata, null);
    verifyZeroInteractions(this.jobCoordinatorListener, this.startpointManager);
}

Example 4 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestStaticResourceJobCoordinator method setupJobCoordinatorMetadata.

/**
 * Set up mocks for {@link JobCoordinatorMetadataManager}.
 * {@code jobMetadataChanges} defines which changes should be detected by the {@link JobCoordinatorMetadataManager}
 */
private JobCoordinatorMetadata setupJobCoordinatorMetadata(JobModel jobModel, Config jobModelConfig, Set<JobMetadataChange> jobMetadataChanges, boolean hasPreviousMetadata) {
    JobCoordinatorMetadata previousMetadata = hasPreviousMetadata ? mock(JobCoordinatorMetadata.class) : null;
    JobCoordinatorMetadata newMetadata = mock(JobCoordinatorMetadata.class);
    when(this.jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, jobModelConfig)).thenReturn(newMetadata);
    when(this.jobCoordinatorMetadataManager.readJobCoordinatorMetadata()).thenReturn(previousMetadata);
    if (hasPreviousMetadata) {
        when(this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata)).thenReturn(jobMetadataChanges);
    } else {
        when(this.jobCoordinatorMetadataManager.checkForMetadataChanges(eq(newMetadata), isNull(JobCoordinatorMetadata.class))).thenReturn(jobMetadataChanges);
    }
    return newMetadata;
}

Example 5 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesMultipleChanges.

@Test
public void testCheckForMetadataChangesMultipleChanges() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate all changes", ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL, JobMetadataChange.CONFIG), metadataChanges);
    assertEquals("New deployment should be 1 since Epoch ID changed", 1, this.jobCoordinatorMetadataManager.getMetrics().getNewDeployment().getValue().intValue());
    assertEquals("Job model changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getJobModelChangedAcrossApplicationAttempt().getValue().intValue());
    assertEquals("Config changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getConfigChangedAcrossApplicationAttempt().getValue().intValue());
}