Examples with StreamRegexMonitor org.apache.samza.coordinator.StreamRegexMonitor used on opensource projects

Example 1 with StreamRegexMonitor

use of org.apache.samza.coordinator.StreamRegexMonitor in project samza by apache.

the class ClusterBasedJobCoordinator method run.

/**
 * Starts the JobCoordinator.
 */
public void run() {
    if (!isStarted.compareAndSet(false, true)) {
        LOG.warn("Attempting to start an already started job coordinator. ");
        return;
    }
    // set up JmxServer (if jmx is enabled)
    if (isJmxEnabled) {
        jmxServer = new JmxServer();
        state.jmxUrl = jmxServer.getJmxUrl();
        state.jmxTunnelingUrl = jmxServer.getTunnelingJmxUrl();
    } else {
        jmxServer = null;
    }
    try {
        // initialize JobCoordinator state
        LOG.info("Starting cluster based job coordinator");
        // write the diagnostics metadata file
        String jobName = new JobConfig(config).getName().get();
        String jobId = new JobConfig(config).getJobId();
        Optional<String> execEnvContainerId = Optional.ofNullable(System.getenv("CONTAINER_ID"));
        DiagnosticsUtil.writeMetadataFile(jobName, jobId, METRICS_SOURCE_NAME, execEnvContainerId, config);
        // create necessary checkpoint and changelog streams, if not created
        JobModel jobModel = jobModelManager.jobModel();
        MetadataResourceUtil metadataResourceUtil = new MetadataResourceUtil(jobModel, this.metrics, config);
        metadataResourceUtil.createResources();
        // create all the resources required for state backend factories
        StorageConfig storageConfig = new StorageConfig(config);
        storageConfig.getBackupFactories().forEach(stateStorageBackendBackupFactory -> {
            StateBackendFactory stateBackendFactory = ReflectionUtil.getObj(stateStorageBackendBackupFactory, StateBackendFactory.class);
            StateBackendAdmin stateBackendAdmin = stateBackendFactory.getAdmin(jobModel, config);
            // Create resources required for state backend admin
            stateBackendAdmin.createResources();
            // Validate resources required for state backend admin
            stateBackendAdmin.validateResources();
        });
        /*
       * We fanout startpoint if and only if
       *  1. Startpoint is enabled in configuration
       *  2. If AM HA is enabled, fanout only if startpoint enabled and job coordinator metadata changed
       */
        if (shouldFanoutStartpoint()) {
            StartpointManager startpointManager = createStartpointManager();
            startpointManager.start();
            try {
                startpointManager.fanOut(JobModelUtil.getTaskToSystemStreamPartitions(jobModel));
            } finally {
                startpointManager.stop();
            }
        }
        // Remap changelog partitions to tasks
        Map<TaskName, Integer> prevPartitionMappings = changelogStreamManager.readPartitionMapping();
        Map<TaskName, Integer> taskPartitionMappings = new HashMap<>();
        Map<String, ContainerModel> containers = jobModel.getContainers();
        for (ContainerModel containerModel : containers.values()) {
            for (TaskModel taskModel : containerModel.getTasks().values()) {
                taskPartitionMappings.put(taskModel.getTaskName(), taskModel.getChangelogPartition().getPartitionId());
            }
        }
        changelogStreamManager.updatePartitionMapping(prevPartitionMappings, taskPartitionMappings);
        containerProcessManager.start();
        systemAdmins.start();
        partitionMonitor.start();
        inputStreamRegexMonitor.ifPresent(StreamRegexMonitor::start);
        // containerPlacementRequestAllocator thread has to start after the cpm is started
        LOG.info("Starting the container placement handler thread");
        containerPlacementMetadataStore.start();
        containerPlacementRequestAllocatorThread.start();
        boolean isInterrupted = false;
        while (!containerProcessManager.shouldShutdown() && !checkAndThrowException() && !isInterrupted && checkcontainerPlacementRequestAllocatorThreadIsAlive()) {
            try {
                Thread.sleep(jobCoordinatorSleepInterval);
            } catch (InterruptedException e) {
                isInterrupted = true;
                LOG.error("Interrupted in job coordinator loop", e);
                Thread.currentThread().interrupt();
            }
        }
    } catch (Throwable e) {
        LOG.error("Exception thrown in the JobCoordinator loop", e);
        throw new SamzaException(e);
    } finally {
        onShutDown();
    }
}

Example 2 with StreamRegexMonitor

use of org.apache.samza.coordinator.StreamRegexMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testSameJobModelAsPrevious.

@Test
public void testSameJobModelAsPrevious() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, null, null);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}

Example 3 with StreamRegexMonitor

use of org.apache.samza.coordinator.StreamRegexMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testNoExistingJobModel.

@Test
public void testNoExistingJobModel() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}

Example 4 with StreamRegexMonitor

use of org.apache.samza.coordinator.StreamRegexMonitor in project samza by apache.

the class TestStaticResourceJobCoordinator method testStopAfterStart.

@Test
public void testStopAfterStart() throws InterruptedException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.copyOf(Arrays.asList(JobMetadataChange.values())), false);
    setUpDiagnosticsManager(jobModel);
    metadataResourceUtil(jobModel);
    // call start in order to set up monitors
    this.staticResourceJobCoordinator.start();
    // call stop to check that the expected components get shut down
    this.staticResourceJobCoordinator.stop();
    verify(this.jobCoordinatorListener).onJobModelExpired();
    verify(this.diagnosticsManager).stop();
    verify(streamPartitionCountMonitor).stop();
    verify(streamRegexMonitor).stop();
    verify(this.coordinatorCommunication).stop();
    verify(this.startpointManager).stop();
    verify(this.systemAdmins).stop();
    verify(this.jobCoordinatorListener).onCoordinatorStop();
}

Example 5 with StreamRegexMonitor

use of org.apache.samza.coordinator.StreamRegexMonitor in project samza by apache.

the class StaticResourceJobCoordinator method jobModelMonitors.

/*
   * Possible race condition: The partition count monitor queries for stream metadata when it is created, so if the
   * partition counts changed between the job model calculation and the creation of the partition count monitor, then
   * the monitor will not trigger an update to the job model. This method should be called right after calculating the
   * job model, in order to reduce the possible time in which a partition count change is missed. This issue also
   * exists in the older ClusterBasedJobCoordinator.
   * TODO This wouldn't be a problem if the partition count monitor used the job model to calculate initial metadata
   */
private JobModelMonitors jobModelMonitors(JobModel jobModel) {
    StreamPartitionCountMonitor streamPartitionCountMonitor = this.streamPartitionCountMonitorFactory.build(jobModel.getConfig(), streamsChanged -> this.jobRestartSignal.restartJob());
    Optional<StreamRegexMonitor> streamRegexMonitor = this.streamRegexMonitorFactory.build(jobModel, jobModel.getConfig(), (initialInputSet, newInputStreams, regexesMonitored) -> this.jobRestartSignal.restartJob());
    return new JobModelMonitors(streamPartitionCountMonitor, streamRegexMonitor.orElse(null));
}