Examples with StreamMetadataCache org.apache.samza.system.StreamMetadataCache used on opensource projects

Example 11 with StreamMetadataCache

use of org.apache.samza.system.StreamMetadataCache in project samza by apache.

the class ClusterBasedJobCoordinator method getPartitionCountMonitor.

private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) {
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    return new StreamPartitionCountMonitorFactory(streamMetadataCache, this.metrics).build(config, streamsChanged -> {
        // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted
        if (hasDurableStores) {
            LOG.error("Input topic partition count changed in a job with durable state. Failing the job. Changed topics: {}", streamsChanged.toString());
            state.status = SamzaApplicationState.SamzaAppStatus.FAILED;
        }
        coordinatorException = new PartitionChangeException("Input topic partition count changes detected for topics: " + streamsChanged.toString());
    });
}

Example 12 with StreamMetadataCache

use of org.apache.samza.system.StreamMetadataCache in project samza by apache.

the class ClusterBasedJobCoordinator method getInputRegexMonitor.

private Optional<StreamRegexMonitor> getInputRegexMonitor(JobModel jobModel, Config config, SystemAdmins systemAdmins) {
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    return new StreamRegexMonitorFactory(streamMetadataCache, this.metrics).build(jobModel, config, (initialInputSet, newInputStreams, regexesMonitored) -> {
        if (hasDurableStores) {
            LOG.error("New input system-streams discovered. Failing the job. New input streams: {}" + " Existing input streams: {}", newInputStreams, initialInputSet);
            state.status = SamzaApplicationState.SamzaAppStatus.FAILED;
        }
        coordinatorException = new InputStreamsDiscoveredException("New input streams discovered: " + newInputStreams);
    });
}

Example 13 with StreamMetadataCache

use of org.apache.samza.system.StreamMetadataCache in project samza by apache.

the class StaticResourceJobCoordinatorFactory method getJobCoordinator.

@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
    JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
    JobConfig jobConfig = new JobConfig(config);
    CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
    CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
    ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
    JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
    Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
    SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
    StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
    StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
    Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
    Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
    return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}