Examples with JobMetadataChange org.apache.samza.job.JobMetadataChange used on opensource projects

Example 1 with JobMetadataChange

use of org.apache.samza.job.JobMetadataChange in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesMultipleChanges.

@Test
public void testCheckForMetadataChangesMultipleChanges() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate all changes", ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL, JobMetadataChange.CONFIG), metadataChanges);
    assertEquals("New deployment should be 1 since Epoch ID changed", 1, this.jobCoordinatorMetadataManager.getMetrics().getNewDeployment().getValue().intValue());
    assertEquals("Job model changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getJobModelChangedAcrossApplicationAttempt().getValue().intValue());
    assertEquals("Config changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getConfigChangedAcrossApplicationAttempt().getValue().intValue());
}

Example 2 with JobMetadataChange

use of org.apache.samza.job.JobMetadataChange in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesJobModelChange.

@Test
public void testCheckForMetadataChangesJobModelChange() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, NEW_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate new job model", ImmutableSet.of(JobMetadataChange.JOB_MODEL), metadataChanges);
    assertEquals("Job model changed across application attempts should be 1", 1, this.jobCoordinatorMetadataManager.getMetrics().getJobModelChangedAcrossApplicationAttempt().getValue().intValue());
    assertEquals("Application attempt count should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getApplicationAttemptCount().getValue().intValue());
}

Example 3 with JobMetadataChange

use of org.apache.samza.job.JobMetadataChange in project samza by apache.

the class StaticResourceJobCoordinator method start.

@Override
public void start() {
    LOG.info("Starting job coordinator");
    this.systemAdmins.start();
    this.startpointManager.ifPresent(StartpointManager::start);
    try {
        JobModel jobModel = newJobModel();
        doSetLoggingContextConfig(jobModel.getConfig());
        // monitors should be created right after job model is calculated (see jobModelMonitors() for more details)
        JobModelMonitors jobModelMonitors = jobModelMonitors(jobModel);
        Optional<DiagnosticsManager> diagnosticsManager = diagnosticsManager(jobModel);
        JobCoordinatorMetadata newMetadata = this.jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, jobModel.getConfig());
        Set<JobMetadataChange> jobMetadataChanges = checkForMetadataChanges(newMetadata);
        if (!jobMetadataChanges.isEmpty() && !jobMetadataChanges.contains(JobMetadataChange.NEW_DEPLOYMENT)) {
            /*
         * If the job coordinator comes up, but not due to a new deployment, and the metadata changed, then trigger a
         * restart. This case applies if the job coordinator died and the job model needed to change while it was down.
         * If there were no metadata changes, then just let the current workers continue to run.
         * If there was a new deployment (which includes the case where the coordinator requested a restart), then we
         * rely on the external resource manager to make sure the previous workers restarted, so we don't need to
         * restart again.
         */
            LOG.info("Triggering job restart");
            this.jobRestartSignal.restartJob();
        } else {
            prepareWorkerExecution(jobModel, newMetadata, jobMetadataChanges);
            // save components that depend on job model in order to manage lifecycle or access later
            this.currentDiagnosticsManager = diagnosticsManager;
            this.currentJobModelMonitors = Optional.of(jobModelMonitors);
            this.currentJobModel = Optional.of(jobModel);
            // lifecycle: start components
            this.coordinatorCommunication.start();
            this.jobCoordinatorListener.ifPresent(listener -> listener.onNewJobModel(this.processorId, jobModel));
            this.currentDiagnosticsManager.ifPresent(DiagnosticsManager::start);
            jobModelMonitors.start();
            this.jobPreparationComplete.set(true);
        }
    } catch (Exception e) {
        LOG.error("Error while running job coordinator; exiting", e);
        throw new SamzaException("Error while running job coordinator", e);
    }
}

Example 4 with JobMetadataChange

use of org.apache.samza.job.JobMetadataChange in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesNewDeployment.

@Test
public void testCheckForMetadataChangesNewDeployment() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate new deployment", ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT), metadataChanges);
    assertEquals("New deployment should be 1 since Epoch ID changed", 1, this.jobCoordinatorMetadataManager.getMetrics().getNewDeployment().getValue().intValue());
    assertEquals("Application attempt count should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getApplicationAttemptCount().getValue().intValue());
}

Example 5 with JobMetadataChange

use of org.apache.samza.job.JobMetadataChange in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesNoPreviousMetadata.

@Test
public void testCheckForMetadataChangesNoPreviousMetadata() {
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, null);
    assertEquals("Metadata check should indicate all changes", ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL, JobMetadataChange.CONFIG), metadataChanges);
    assertEquals("New deployment should be 1 since Epoch ID changed", 1, this.jobCoordinatorMetadataManager.getMetrics().getNewDeployment().getValue().intValue());
    assertEquals("Job model changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getJobModelChangedAcrossApplicationAttempt().getValue().intValue());
    assertEquals("Config changed across application attempts should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getConfigChangedAcrossApplicationAttempt().getValue().intValue());
}