use of org.apache.samza.container.LocalityManager in project samza by apache.
the class SamzaTaskProxy method getJobModel.
/**
* Retrieves the jobModel from the jobCoordinator.
* @param jobInstance the job instance (jobId, jobName).
* @return the JobModel fetched from the coordinator stream.
*/
protected JobModel getJobModel(JobInstance jobInstance) {
CoordinatorStreamSystemConsumer coordinatorSystemConsumer = null;
CoordinatorStreamSystemProducer coordinatorSystemProducer = null;
try {
CoordinatorStreamSystemFactory coordinatorStreamSystemFactory = new CoordinatorStreamSystemFactory();
Config coordinatorSystemConfig = getCoordinatorSystemConfig(jobInstance);
LOG.info("Using config: {} to create coordinatorStream producer and consumer.", coordinatorSystemConfig);
coordinatorSystemConsumer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(coordinatorSystemConfig, METRICS_REGISTRY);
coordinatorSystemProducer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(coordinatorSystemConfig, METRICS_REGISTRY);
LOG.info("Registering coordinator system stream consumer.");
coordinatorSystemConsumer.register();
LOG.debug("Starting coordinator system stream consumer.");
coordinatorSystemConsumer.start();
LOG.debug("Bootstrapping coordinator system stream consumer.");
coordinatorSystemConsumer.bootstrap();
LOG.info("Registering coordinator system stream producer.");
coordinatorSystemProducer.register(SOURCE);
Config config = coordinatorSystemConsumer.getConfig();
LOG.info("Got config from coordinatorSystemConsumer: {}.", config);
ChangelogPartitionManager changelogManager = new ChangelogPartitionManager(coordinatorSystemProducer, coordinatorSystemConsumer, SOURCE);
changelogManager.start();
LocalityManager localityManager = new LocalityManager(coordinatorSystemProducer, coordinatorSystemConsumer);
localityManager.start();
String jobCoordinatorSystemName = config.get(JobConfig.JOB_COORDINATOR_SYSTEM());
/**
* Select job coordinator system properties from config and instantiate SystemAdmin for it alone.
* Instantiating SystemAdmin's for other input/output systems defined in config is unnecessary.
*/
Config systemAdminConfig = config.subset(String.format("systems.%s", jobCoordinatorSystemName), false);
scala.collection.immutable.Map<String, SystemAdmin> systemAdmins = JobModelManager.getSystemAdmins(systemAdminConfig);
StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
Map<TaskName, Integer> changeLogPartitionMapping = changelogManager.readChangeLogPartitionMapping();
return JobModelManager.readJobModel(config, changeLogPartitionMapping, localityManager, streamMetadataCache, null);
} finally {
if (coordinatorSystemConsumer != null) {
coordinatorSystemConsumer.stop();
}
if (coordinatorSystemProducer != null) {
coordinatorSystemProducer.stop();
}
}
}
use of org.apache.samza.container.LocalityManager in project samza by apache.
the class TestContainerAllocatorWithHostAffinity method setup.
@Before
public void setup() throws Exception {
LocalityManager mockLocalityManager = mock(LocalityManager.class);
when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of("0", new ProcessorLocality("0", "abc"))));
CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(config);
CoordinatorStreamStore coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
coordinatorStreamStore.init();
containerPlacementMetadataStore = new ContainerPlacementMetadataStore(coordinatorStreamStore);
containerPlacementMetadataStore.start();
containerManager = new ContainerManager(containerPlacementMetadataStore, state, clusterResourceManager, true, false, mockLocalityManager, faultDomainManager, config);
containerAllocator = new ContainerAllocator(clusterResourceManager, config, state, true, containerManager);
requestState = new MockContainerRequestState(clusterResourceManager, true);
Field requestStateField = containerAllocator.getClass().getDeclaredField("resourceRequestState");
requestStateField.setAccessible(true);
requestStateField.set(containerAllocator, requestState);
allocatorThread = new Thread(containerAllocator);
}
use of org.apache.samza.container.LocalityManager in project samza by apache.
the class YarnJobValidationTool method validateJmxMetrics.
public void validateJmxMetrics() throws Exception {
MetricsRegistry metricsRegistry = new MetricsRegistryMap();
CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(config, metricsRegistry);
coordinatorStreamStore.init();
try {
LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE));
validator.init(config);
LocalityModel localityModel = localityManager.readLocality();
for (ProcessorLocality processorLocality : localityModel.getProcessorLocalities().values()) {
String containerId = processorLocality.id();
String jmxUrl = processorLocality.jmxTunnelingUrl();
if (StringUtils.isNotBlank(jmxUrl)) {
log.info("validate container " + containerId + " metrics with JMX: " + jmxUrl);
JmxMetricsAccessor jmxMetrics = new JmxMetricsAccessor(jmxUrl);
jmxMetrics.connect();
validator.validate(jmxMetrics);
jmxMetrics.close();
log.info("validate container " + containerId + " successfully");
}
}
validator.complete();
} finally {
coordinatorStreamStore.close();
}
}
use of org.apache.samza.container.LocalityManager in project samza by apache.
the class JobModelHelper method getGrouperMetadata.
private GrouperMetadata getGrouperMetadata(Config config, LocalityManager localityManager, TaskAssignmentManager taskAssignmentManager, TaskPartitionAssignmentManager taskPartitionAssignmentManager) {
Map<String, LocationId> processorLocality = getProcessorLocality(config, localityManager);
Map<TaskName, TaskMode> taskModes = taskAssignmentManager.readTaskModes();
Map<TaskName, String> taskNameToProcessorId = new HashMap<>();
Map<TaskName, LocationId> taskLocality = new HashMap<>();
// We read the taskAssignment only for ActiveTasks, i.e., tasks that have no task-mode or have an active task mode
taskAssignmentManager.readTaskAssignment().forEach((taskNameString, containerId) -> {
TaskName taskName = new TaskName(taskNameString);
if (isActiveTask(taskName, taskModes)) {
taskNameToProcessorId.put(taskName, containerId);
if (processorLocality.containsKey(containerId)) {
taskLocality.put(taskName, processorLocality.get(containerId));
}
}
});
Map<SystemStreamPartition, List<String>> sspToTaskMapping = taskPartitionAssignmentManager.readTaskPartitionAssignments();
Map<TaskName, List<SystemStreamPartition>> taskPartitionAssignments = new HashMap<>();
// Task to partition assignments is stored as {@see SystemStreamPartition} to list of {@see TaskName} in
// coordinator stream. This is done due to the 1 MB value size limit in a kafka topic. Conversion to
// taskName to SystemStreamPartitions is done here to wire-in the data to {@see JobModel}.
sspToTaskMapping.forEach((systemStreamPartition, taskNames) -> taskNames.forEach(taskNameString -> {
TaskName taskName = new TaskName(taskNameString);
if (isActiveTask(taskName, taskModes)) {
taskPartitionAssignments.putIfAbsent(taskName, new ArrayList<>());
taskPartitionAssignments.get(taskName).add(systemStreamPartition);
}
}));
return new GrouperMetadataImpl(processorLocality, taskLocality, taskPartitionAssignments, taskNameToProcessorId);
}
use of org.apache.samza.container.LocalityManager in project samza by apache.
the class TestContainerAllocatorWithoutHostAffinity method setup.
@Before
public void setup() throws Exception {
LocalityManager mockLocalityManager = mock(LocalityManager.class);
when(mockLocalityManager.readLocality()).thenReturn(new LocalityModel(new HashMap<>()));
CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(config);
CoordinatorStreamStore coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
coordinatorStreamStore.init();
containerPlacementMetadataStore = new ContainerPlacementMetadataStore(coordinatorStreamStore);
containerPlacementMetadataStore.start();
containerAllocator = new ContainerAllocator(manager, config, state, false, new ContainerManager(containerPlacementMetadataStore, state, manager, false, false, mockLocalityManager, faultDomainManager, config));
requestState = new MockContainerRequestState(manager, false);
Field requestStateField = containerAllocator.getClass().getDeclaredField("resourceRequestState");
requestStateField.setAccessible(true);
requestStateField.set(containerAllocator, requestState);
allocatorThread = new Thread(containerAllocator);
}
Aggregations