Examples with NamespaceAwareCoordinatorStreamStore org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore used on opensource projects

Example 6 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class StaticResourceJobCoordinatorFactory method buildJobModelHelper.

private static JobModelHelper buildJobModelHelper(MetadataStore metadataStore, StreamMetadataCache streamMetadataCache) {
    LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetContainerHostMapping.TYPE));
    TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetTaskContainerMapping.TYPE), new NamespaceAwareCoordinatorStreamStore(metadataStore, SetTaskModeMapping.TYPE));
    TaskPartitionAssignmentManager taskPartitionAssignmentManager = new TaskPartitionAssignmentManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetTaskPartitionMapping.TYPE));
    return new JobModelHelper(localityManager, taskAssignmentManager, taskPartitionAssignmentManager, streamMetadataCache, JobModelCalculator.INSTANCE);
}

Example 7 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class YarnJobValidationTool method validateJmxMetrics.

public void validateJmxMetrics() throws Exception {
    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(config, metricsRegistry);
    coordinatorStreamStore.init();
    try {
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE));
        validator.init(config);
        LocalityModel localityModel = localityManager.readLocality();
        for (ProcessorLocality processorLocality : localityModel.getProcessorLocalities().values()) {
            String containerId = processorLocality.id();
            String jmxUrl = processorLocality.jmxTunnelingUrl();
            if (StringUtils.isNotBlank(jmxUrl)) {
                log.info("validate container " + containerId + " metrics with JMX: " + jmxUrl);
                JmxMetricsAccessor jmxMetrics = new JmxMetricsAccessor(jmxUrl);
                jmxMetrics.connect();
                validator.validate(jmxMetrics);
                jmxMetrics.close();
                log.info("validate container " + containerId + " successfully");
            }
        }
        validator.complete();
    } finally {
        coordinatorStreamStore.close();
    }
}

Example 8 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class StaticResourceJobCoordinatorFactory method getJobCoordinator.

@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
    JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
    JobConfig jobConfig = new JobConfig(config);
    CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
    CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
    ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
    JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
    Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
    SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
    StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
    StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
    Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
    Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
    return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}

Example 9 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class TestLocalityManager method testLocalityManager.

@Test
public void testLocalityManager() {
    LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetContainerHostMapping.TYPE));
    localityManager.writeContainerToHostMapping("0", "localhost");
    Map<String, Map<String, String>> localMap = readContainerLocality(localityManager);
    Map<String, Map<String, String>> expectedMap = new HashMap<String, Map<String, String>>() {

        {
            this.put("0", new HashMap<String, String>() {

                {
                    this.put(SetContainerHostMapping.HOST_KEY, "localhost");
                }
            });
        }
    };
    assertEquals(expectedMap, localMap);
    localityManager.close();
    MockCoordinatorStreamSystemProducer producer = coordinatorStreamStoreTestUtil.getMockCoordinatorStreamSystemProducer();
    MockCoordinatorStreamSystemConsumer consumer = coordinatorStreamStoreTestUtil.getMockCoordinatorStreamSystemConsumer();
    assertTrue(producer.isStopped());
    assertTrue(consumer.isStopped());
}

Example 10 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class TestLocalityManager method testWriteOnlyLocalityManager.

@Test
public void testWriteOnlyLocalityManager() {
    LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetContainerHostMapping.TYPE));
    localityManager.writeContainerToHostMapping("1", "localhost");
    assertEquals(readContainerLocality(localityManager).size(), 1);
    assertEquals(ImmutableMap.of("1", ImmutableMap.of("host", "localhost")), readContainerLocality(localityManager));
    localityManager.close();
    MockCoordinatorStreamSystemProducer producer = coordinatorStreamStoreTestUtil.getMockCoordinatorStreamSystemProducer();
    MockCoordinatorStreamSystemConsumer consumer = coordinatorStreamStoreTestUtil.getMockCoordinatorStreamSystemConsumer();
    assertTrue(producer.isStopped());
    assertTrue(consumer.isStopped());
}