Examples with CoordinatorStreamSystemConsumer org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer used on opensource projects

Example 1 with CoordinatorStreamSystemConsumer

use of org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer in project samza by apache.

the class SamzaTaskProxy method getJobModel.

/**
   * Retrieves the jobModel from the jobCoordinator.
   * @param jobInstance the job instance (jobId, jobName).
   * @return the JobModel fetched from the coordinator stream.
   */
protected JobModel getJobModel(JobInstance jobInstance) {
    CoordinatorStreamSystemConsumer coordinatorSystemConsumer = null;
    CoordinatorStreamSystemProducer coordinatorSystemProducer = null;
    try {
        CoordinatorStreamSystemFactory coordinatorStreamSystemFactory = new CoordinatorStreamSystemFactory();
        Config coordinatorSystemConfig = getCoordinatorSystemConfig(jobInstance);
        LOG.info("Using config: {} to create coordinatorStream producer and consumer.", coordinatorSystemConfig);
        coordinatorSystemConsumer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(coordinatorSystemConfig, METRICS_REGISTRY);
        coordinatorSystemProducer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(coordinatorSystemConfig, METRICS_REGISTRY);
        LOG.info("Registering coordinator system stream consumer.");
        coordinatorSystemConsumer.register();
        LOG.debug("Starting coordinator system stream consumer.");
        coordinatorSystemConsumer.start();
        LOG.debug("Bootstrapping coordinator system stream consumer.");
        coordinatorSystemConsumer.bootstrap();
        LOG.info("Registering coordinator system stream producer.");
        coordinatorSystemProducer.register(SOURCE);
        Config config = coordinatorSystemConsumer.getConfig();
        LOG.info("Got config from coordinatorSystemConsumer: {}.", config);
        ChangelogPartitionManager changelogManager = new ChangelogPartitionManager(coordinatorSystemProducer, coordinatorSystemConsumer, SOURCE);
        changelogManager.start();
        LocalityManager localityManager = new LocalityManager(coordinatorSystemProducer, coordinatorSystemConsumer);
        localityManager.start();
        String jobCoordinatorSystemName = config.get(JobConfig.JOB_COORDINATOR_SYSTEM());
        /**
       * Select job coordinator system properties from config and instantiate SystemAdmin for it alone.
       * Instantiating SystemAdmin's for other input/output systems defined in config is unnecessary.
       */
        Config systemAdminConfig = config.subset(String.format("systems.%s", jobCoordinatorSystemName), false);
        scala.collection.immutable.Map<String, SystemAdmin> systemAdmins = JobModelManager.getSystemAdmins(systemAdminConfig);
        StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
        Map<TaskName, Integer> changeLogPartitionMapping = changelogManager.readChangeLogPartitionMapping();
        return JobModelManager.readJobModel(config, changeLogPartitionMapping, localityManager, streamMetadataCache, null);
    } finally {
        if (coordinatorSystemConsumer != null) {
            coordinatorSystemConsumer.stop();
        }
        if (coordinatorSystemProducer != null) {
            coordinatorSystemProducer.stop();
        }
    }
}

Example 2 with CoordinatorStreamSystemConsumer

use of org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer in project samza by apache.

the class RemoteJobPlanner method getConfigFromPrevRun.

private Config getConfigFromPrevRun() {
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap());
    consumer.register();
    consumer.start();
    consumer.bootstrap();
    consumer.stop();
    Config cfg = consumer.getConfig();
    LOG.info("Previous config is: " + cfg.toString());
    return cfg;
}

Example 3 with CoordinatorStreamSystemConsumer

use of org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer in project samza by apache.

the class SamzaTaskProxy method readTasksFromCoordinatorStream.

/**
 * Builds list of {@link Task} from job model in coordinator stream.
 * @param consumer system consumer associated with a job's coordinator stream.
 * @return list of {@link Task} constructed from job model in coordinator stream.
 */
protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) {
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(consumer.getConfig(), new MetricsRegistryMap());
    LocalityManager localityManager = new LocalityManager(coordinatorStreamStore);
    Map<String, ProcessorLocality> containerLocalities = localityManager.readLocality().getProcessorLocalities();
    TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetTaskContainerMapping.TYPE), new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetTaskModeMapping.TYPE));
    Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment();
    StorageConfig storageConfig = new StorageConfig(consumer.getConfig());
    List<String> storeNames = storageConfig.getStoreNames();
    return taskNameToContainerIdMapping.entrySet().stream().map(entry -> {
        String hostName = Optional.ofNullable(containerLocalities.get(entry.getValue())).map(ProcessorLocality::host).orElse(null);
        return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames);
    }).collect(Collectors.toList());
}

Example 4 with CoordinatorStreamSystemConsumer

use of org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer in project samza by apache.

the class SamzaTaskProxy method getTasks.

/**
 * Fetches the complete job model from the coordinator stream based upon the provided {@link JobInstance}
 * param, transforms it to a list of {@link Task} and returns it.
 * {@inheritDoc}
 */
@Override
public List<Task> getTasks(JobInstance jobInstance) throws IOException, InterruptedException {
    Preconditions.checkArgument(installFinder.isInstalled(jobInstance), String.format("Invalid job instance : %s", jobInstance));
    CoordinatorStreamSystemConsumer coordinatorStreamSystemConsumer = null;
    try {
        coordinatorStreamSystemConsumer = initializeCoordinatorStreamConsumer(jobInstance);
        return readTasksFromCoordinatorStream(coordinatorStreamSystemConsumer);
    } finally {
        if (coordinatorStreamSystemConsumer != null) {
            coordinatorStreamSystemConsumer.stop();
        }
    }
}

Example 5 with CoordinatorStreamSystemConsumer

use of org.apache.samza.coordinator.stream.CoordinatorStreamSystemConsumer in project samza by apache.

the class SamzaTaskProxy method initializeCoordinatorStreamConsumer.

/**
 * Initialize {@link CoordinatorStreamSystemConsumer} based upon {@link JobInstance} parameter.
 * @param jobInstance the job instance to get CoordinatorStreamSystemConsumer for.
 * @return built and initialized CoordinatorStreamSystemConsumer.
 */
protected CoordinatorStreamSystemConsumer initializeCoordinatorStreamConsumer(JobInstance jobInstance) {
    Config coordinatorSystemConfig = getCoordinatorSystemConfig(jobInstance);
    LOG.debug("Using config: {} to create coordinatorStream consumer.", coordinatorSystemConfig);
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(coordinatorSystemConfig, METRICS_REGISTRY);
    LOG.debug("Registering coordinator system stream consumer.");
    consumer.register();
    LOG.debug("Starting coordinator system stream consumer.");
    consumer.start();
    LOG.debug("Bootstrapping coordinator system stream consumer.");
    consumer.bootstrap();
    return consumer;
}