Examples with NamespaceAwareCoordinatorStreamStore org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore used on opensource projects

Example 1 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class TestExecutionContainerIdManager method setup.

@Before
public void setup() {
    coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(CONFIG);
    coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
    store = Mockito.spy(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetExecutionEnvContainerIdMapping.TYPE));
    executionContainerIdManager = new ExecutionContainerIdManager(store);
}

Example 2 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class ZkJobCoordinator method loadMetadataResources.

/**
 * Stores the configuration of the job in the coordinator stream.
 */
@VisibleForTesting
void loadMetadataResources(JobModel jobModel) {
    try {
        MetadataResourceUtil metadataResourceUtil = createMetadataResourceUtil(jobModel, config);
        metadataResourceUtil.createResources();
        if (coordinatorStreamStore != null) {
            // TODO: SAMZA-2273 - publish configs async
            CoordinatorStreamValueSerde jsonSerde = new CoordinatorStreamValueSerde(SetConfig.TYPE);
            NamespaceAwareCoordinatorStreamStore configStore = new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE);
            for (Map.Entry<String, String> entry : config.entrySet()) {
                byte[] serializedValue = jsonSerde.toBytes(entry.getValue());
                configStore.put(entry.getKey(), serializedValue);
            }
            configStore.flush();
            if (new JobConfig(config).getStartpointEnabled()) {
                // fan out the startpoints
                StartpointManager startpointManager = createStartpointManager();
                startpointManager.start();
                try {
                    startpointManager.fanOut(JobModelUtil.getTaskToSystemStreamPartitions(jobModel));
                } finally {
                    startpointManager.stop();
                }
            }
        } else {
            LOG.warn("No metadata store registered to this job coordinator. Config not written to the metadata store and no Startpoints fan out.");
        }
    } catch (IOException ex) {
        throw new SamzaException(String.format("IO exception while loading metadata resources."), ex);
    }
}

Example 3 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class ContainerLaunchUtil method run.

@VisibleForTesting
static void run(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, String jobName, String jobId, String containerId, Optional<String> executionEnvContainerId, Optional<String> samzaEpochId, JobModel jobModel, Config config, Optional<ExternalContext> externalContextOptional) {
    CoordinatorStreamStore coordinatorStreamStore = buildCoordinatorStreamStore(config, new MetricsRegistryMap());
    coordinatorStreamStore.init();
    /*
     * We track the exit code and only trigger exit in the finally block to make sure we are able to execute all the
     * clean up steps. Prior implementation had short circuited exit causing some of the clean up steps to be missed.
     */
    int exitCode = 0;
    try {
        TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc);
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetContainerHostMapping.TYPE));
        // StartpointManager wraps the coordinatorStreamStore in the namespaces internally
        StartpointManager startpointManager = null;
        if (new JobConfig(config).getStartpointEnabled()) {
            startpointManager = new StartpointManager(coordinatorStreamStore);
        }
        Map<String, MetricsReporter> metricsReporters = loadMetricsReporters(appDesc, containerId, config);
        // Creating diagnostics manager and reporter, and wiring it respectively
        Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(jobName, jobId, jobModel, containerId, executionEnvContainerId, samzaEpochId, config);
        MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
        SamzaContainer container = SamzaContainer$.MODULE$.apply(containerId, jobModel, ScalaJavaUtil.toScalaMap(metricsReporters), metricsRegistryMap, taskFactory, JobContextImpl.fromConfigWithDefaults(config, jobModel), Option.apply(appDesc.getApplicationContainerContextFactory().orElse(null)), Option.apply(appDesc.getApplicationTaskContextFactory().orElse(null)), Option.apply(externalContextOptional.orElse(null)), localityManager, startpointManager, Option.apply(diagnosticsManager.orElse(null)));
        ProcessorLifecycleListener processorLifecycleListener = appDesc.getProcessorLifecycleListenerFactory().createInstance(new ProcessorContext() {
        }, config);
        ClusterBasedProcessorLifecycleListener listener = new ClusterBasedProcessorLifecycleListener(config, processorLifecycleListener, container::shutdown);
        container.setContainerListener(listener);
        ContainerHeartbeatMonitor heartbeatMonitor = createContainerHeartbeatMonitor(container, new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE), config);
        if (heartbeatMonitor != null) {
            heartbeatMonitor.start();
        }
        if (new JobConfig(config).getApplicationMasterHighAvailabilityEnabled()) {
            executionEnvContainerId.ifPresent(execEnvContainerId -> {
                ExecutionContainerIdManager executionContainerIdManager = new ExecutionContainerIdManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetExecutionEnvContainerIdMapping.TYPE));
                executionContainerIdManager.writeExecutionEnvironmentContainerIdMapping(containerId, execEnvContainerId);
            });
        }
        container.run();
        if (heartbeatMonitor != null) {
            heartbeatMonitor.stop();
        }
        // overriding the value with what the listener returns
        if (containerRunnerException == null) {
            containerRunnerException = listener.getContainerException();
        }
        if (containerRunnerException != null) {
            log.error("Container stopped with Exception. Exiting process now.", containerRunnerException);
            exitCode = 1;
        }
    } catch (Throwable e) {
        /*
       * Two separate log statements are intended to print the entire stack trace as part of the logs. Using
       * single log statement with custom format requires explicitly fetching stack trace and null checks which makes
       * the code slightly hard to read in comparison with the current choice.
       */
        log.error("Exiting the process due to", e);
        log.error("Container runner exception: ", containerRunnerException);
        exitCode = 1;
    } finally {
        coordinatorStreamStore.close();
        /*
       * Only exit in the scenario of non-zero exit code in order to maintain parity with current implementation where
       * the method completes when no errors are encountered.
       */
        if (exitCode != 0) {
            exitProcess(exitCode);
        }
    }
}

Example 4 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class TestTaskPartitionAssignmentManager method setup.

@Before
public void setup() {
    CoordinatorStreamStoreTestUtil coordinatorStreamStoreTestUtil = new CoordinatorStreamStoreTestUtil(CONFIG);
    CoordinatorStreamStore coordinatorStreamStore = coordinatorStreamStoreTestUtil.getCoordinatorStreamStore();
    taskPartitionAssignmentManager = new TaskPartitionAssignmentManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetTaskPartitionMapping.TYPE));
}

Example 5 with NamespaceAwareCoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore in project samza by apache.

the class TestJobCoordinatorMetadataManager method setup.

@Before
public void setup() {
    Map<TaskName, TaskModel> tasksForContainer1 = ImmutableMap.of(new TaskName("t1"), new TaskModel(new TaskName("t1"), ImmutableSet.of(), new Partition(0)), new TaskName("t2"), new TaskModel(new TaskName("t2"), ImmutableSet.of(), new Partition(1)));
    Map<TaskName, TaskModel> tasksForContainer2 = ImmutableMap.of(new TaskName("t3"), new TaskModel(new TaskName("t3"), ImmutableSet.of(), new Partition(2)), new TaskName("t4"), new TaskModel(new TaskName("t4"), ImmutableSet.of(), new Partition(3)), new TaskName("t5"), new TaskModel(new TaskName("t5"), ImmutableSet.of(), new Partition(4)));
    ContainerModel containerModel1 = new ContainerModel("0", tasksForContainer1);
    ContainerModel containerModel2 = new ContainerModel("1", tasksForContainer2);
    containerModelMap = ImmutableMap.of("0", containerModel1, "1", containerModel2);
    CoordinatorStreamStoreTestUtil mockCoordinatorStreamStore = new CoordinatorStreamStoreTestUtil(COORDINATOR_STORE_CONFIG);
    metadataStore = spy(new NamespaceAwareCoordinatorStreamStore(mockCoordinatorStreamStore.getCoordinatorStreamStore(), SetJobCoordinatorMetadataMessage.TYPE));
    jobCoordinatorMetadataManager = spy(new JobCoordinatorMetadataManager(metadataStore, ClusterType.YARN, new MetricsRegistryMap()));
}