Search in sources :

Example 1 with ChangelogStreamManager

use of org.apache.samza.storage.ChangelogStreamManager in project samza by apache.

the class StaticResourceJobCoordinatorFactory method getJobCoordinator.

@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
    JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
    JobConfig jobConfig = new JobConfig(config);
    CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
    CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
    ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
    JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
    Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
    SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
    StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
    StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
    Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
    Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
    return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}
Also used : StreamMetadataCache(org.apache.samza.system.StreamMetadataCache) CoordinatorCommunicationContext(org.apache.samza.coordinator.communication.CoordinatorCommunicationContext) JobCoordinatorMetadataManager(org.apache.samza.job.metadata.JobCoordinatorMetadataManager) StreamPartitionCountMonitorFactory(org.apache.samza.coordinator.StreamPartitionCountMonitorFactory) JobInfoServingContext(org.apache.samza.coordinator.communication.JobInfoServingContext) JobConfig(org.apache.samza.config.JobConfig) StreamRegexMonitorFactory(org.apache.samza.coordinator.StreamRegexMonitorFactory) NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) CoordinatorCommunication(org.apache.samza.coordinator.communication.CoordinatorCommunication) HttpCoordinatorToWorkerCommunicationFactory(org.apache.samza.coordinator.communication.HttpCoordinatorToWorkerCommunicationFactory) JobRestartSignalFactory(org.apache.samza.coordinator.lifecycle.JobRestartSignalFactory) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) JobRestartSignalFactoryContext(org.apache.samza.coordinator.lifecycle.JobRestartSignalFactoryContext) StartpointManager(org.apache.samza.startpoint.StartpointManager) JobModelHelper(org.apache.samza.coordinator.JobModelHelper) JobRestartSignal(org.apache.samza.coordinator.lifecycle.JobRestartSignal) SystemAdmins(org.apache.samza.system.SystemAdmins) ChangelogStreamManager(org.apache.samza.storage.ChangelogStreamManager)

Aggregations

JobConfig (org.apache.samza.config.JobConfig)1 JobCoordinatorConfig (org.apache.samza.config.JobCoordinatorConfig)1 JobModelHelper (org.apache.samza.coordinator.JobModelHelper)1 StreamPartitionCountMonitorFactory (org.apache.samza.coordinator.StreamPartitionCountMonitorFactory)1 StreamRegexMonitorFactory (org.apache.samza.coordinator.StreamRegexMonitorFactory)1 CoordinatorCommunication (org.apache.samza.coordinator.communication.CoordinatorCommunication)1 CoordinatorCommunicationContext (org.apache.samza.coordinator.communication.CoordinatorCommunicationContext)1 HttpCoordinatorToWorkerCommunicationFactory (org.apache.samza.coordinator.communication.HttpCoordinatorToWorkerCommunicationFactory)1 JobInfoServingContext (org.apache.samza.coordinator.communication.JobInfoServingContext)1 JobRestartSignal (org.apache.samza.coordinator.lifecycle.JobRestartSignal)1 JobRestartSignalFactory (org.apache.samza.coordinator.lifecycle.JobRestartSignalFactory)1 JobRestartSignalFactoryContext (org.apache.samza.coordinator.lifecycle.JobRestartSignalFactoryContext)1 NamespaceAwareCoordinatorStreamStore (org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore)1 JobCoordinatorMetadataManager (org.apache.samza.job.metadata.JobCoordinatorMetadataManager)1 StartpointManager (org.apache.samza.startpoint.StartpointManager)1 ChangelogStreamManager (org.apache.samza.storage.ChangelogStreamManager)1 StreamMetadataCache (org.apache.samza.system.StreamMetadataCache)1 SystemAdmins (org.apache.samza.system.SystemAdmins)1