Examples with JobCoordinatorMetadataManager org.apache.samza.job.metadata.JobCoordinatorMetadataManager used on opensource projects

Example 1 with JobCoordinatorMetadataManager

use of org.apache.samza.job.metadata.JobCoordinatorMetadataManager in project samza by apache.

the class ClusterBasedJobCoordinator method generateAndUpdateJobCoordinatorMetadata.

/**
 * Generate the job coordinator metadata for current application attempt and checks for changes in the
 * metadata from the previous attempt and writes the updates metadata to coordinator stream.
 *
 * @param jobModel job model used to generate the job coordinator metadata
 */
@VisibleForTesting
void generateAndUpdateJobCoordinatorMetadata(JobModel jobModel) {
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = createJobCoordinatorMetadataManager();
    JobCoordinatorMetadata previousMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    JobCoordinatorMetadata newMetadata = jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, config);
    if (!jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata).isEmpty()) {
        jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(newMetadata);
        metadataChangedAcrossAttempts = true;
    }
}

Example 2 with JobCoordinatorMetadataManager

use of org.apache.samza.job.metadata.JobCoordinatorMetadataManager in project samza by apache.

the class TestClusterBasedJobCoordinator method testGenerateAndUpdateJobCoordinatorMetadata.

@Test
public void testGenerateAndUpdateJobCoordinatorMetadata() {
    Config jobConfig = new MapConfig(configMap);
    when(CoordinatorStreamUtil.readConfigFromCoordinatorStream(anyObject())).thenReturn(jobConfig);
    ClusterBasedJobCoordinator clusterBasedJobCoordinator = spy(ClusterBasedJobCoordinatorRunner.createFromMetadataStore(jobConfig));
    JobCoordinatorMetadata previousMetadata = mock(JobCoordinatorMetadata.class);
    JobCoordinatorMetadata newMetadata = mock(JobCoordinatorMetadata.class);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = mock(JobCoordinatorMetadataManager.class);
    JobModel mockJobModel = mock(JobModel.class);
    when(jobCoordinatorMetadataManager.readJobCoordinatorMetadata()).thenReturn(previousMetadata);
    when(jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(any(), any())).thenReturn(newMetadata);
    when(jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata)).thenReturn(ImmutableSet.of());
    when(clusterBasedJobCoordinator.createJobCoordinatorMetadataManager()).thenReturn(jobCoordinatorMetadataManager);
    /*
     * Verify if there are no changes to metadata, the metadata changed flag remains false and no interactions
     * with job coordinator metadata manager
     */
    clusterBasedJobCoordinator.generateAndUpdateJobCoordinatorMetadata(mockJobModel);
    assertFalse("JC metadata changed should remain unchanged", clusterBasedJobCoordinator.isMetadataChangedAcrossAttempts());
    verify(jobCoordinatorMetadataManager, times(0)).writeJobCoordinatorMetadata(any());
    /*
     * Verify if there are changes to metadata, we persist the new metadata & update the metadata changed flag
     */
    when(jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata)).thenReturn(ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT));
    clusterBasedJobCoordinator.generateAndUpdateJobCoordinatorMetadata(mockJobModel);
    assertTrue("JC metadata changed should be true", clusterBasedJobCoordinator.isMetadataChangedAcrossAttempts());
    verify(jobCoordinatorMetadataManager, times(1)).writeJobCoordinatorMetadata(newMetadata);
}

Example 3 with JobCoordinatorMetadataManager

use of org.apache.samza.job.metadata.JobCoordinatorMetadataManager in project samza by apache.

the class StaticResourceJobCoordinatorFactory method getJobCoordinator.

@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
    JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
    JobConfig jobConfig = new JobConfig(config);
    CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
    CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
    ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
    JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
    Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
    SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
    StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
    StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
    Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
    Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
    return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}