Examples with JobCoordinatorMetadata org.apache.samza.job.JobCoordinatorMetadata used on opensource projects

Example 6 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesJobModelChange.

@Test
public void testCheckForMetadataChangesJobModelChange() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, NEW_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate new job model", ImmutableSet.of(JobMetadataChange.JOB_MODEL), metadataChanges);
    assertEquals("Job model changed across application attempts should be 1", 1, this.jobCoordinatorMetadataManager.getMetrics().getJobModelChangedAcrossApplicationAttempt().getValue().intValue());
    assertEquals("Application attempt count should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getApplicationAttemptCount().getValue().intValue());
}

Example 7 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestJobCoordinatorMetadataManager method testReadJobCoordinatorMetadataFailed.

@Test
public void testReadJobCoordinatorMetadataFailed() {
    JobCoordinatorMetadata jobCoordinatorMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Serde<String> mockSerde = spy(new CoordinatorStreamValueSerde(SetJobCoordinatorMetadataMessage.TYPE));
    doThrow(new RuntimeException("Failed to read coordinator stream")).when(mockSerde).fromBytes(any());
    jobCoordinatorMetadataManager = spy(new JobCoordinatorMetadataManager(metadataStore, ClusterType.YARN, new MetricsRegistryMap(), mockSerde));
    jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(jobCoordinatorMetadata);
    JobCoordinatorMetadata actualMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    assertNull("Read failed should return null", actualMetadata);
    assertEquals("Metadata read failed count should be 1", 1, jobCoordinatorMetadataManager.getMetrics().getMetadataReadFailedCount().getValue().intValue());
}

Example 8 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestJobCoordinatorMetadataManager method testReadWriteJobCoordinatorMetadata.

@Test
public void testReadWriteJobCoordinatorMetadata() {
    JobCoordinatorMetadata jobCoordinatorMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(jobCoordinatorMetadata);
    JobCoordinatorMetadata actualJobCoordinatorMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    assertEquals("Mismatch in job coordinator metadata", jobCoordinatorMetadata, actualJobCoordinatorMetadata);
}

Example 9 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class StaticResourceJobCoordinator method start.

@Override
public void start() {
    LOG.info("Starting job coordinator");
    this.systemAdmins.start();
    this.startpointManager.ifPresent(StartpointManager::start);
    try {
        JobModel jobModel = newJobModel();
        doSetLoggingContextConfig(jobModel.getConfig());
        // monitors should be created right after job model is calculated (see jobModelMonitors() for more details)
        JobModelMonitors jobModelMonitors = jobModelMonitors(jobModel);
        Optional<DiagnosticsManager> diagnosticsManager = diagnosticsManager(jobModel);
        JobCoordinatorMetadata newMetadata = this.jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, jobModel.getConfig());
        Set<JobMetadataChange> jobMetadataChanges = checkForMetadataChanges(newMetadata);
        if (!jobMetadataChanges.isEmpty() && !jobMetadataChanges.contains(JobMetadataChange.NEW_DEPLOYMENT)) {
            /*
         * If the job coordinator comes up, but not due to a new deployment, and the metadata changed, then trigger a
         * restart. This case applies if the job coordinator died and the job model needed to change while it was down.
         * If there were no metadata changes, then just let the current workers continue to run.
         * If there was a new deployment (which includes the case where the coordinator requested a restart), then we
         * rely on the external resource manager to make sure the previous workers restarted, so we don't need to
         * restart again.
         */
            LOG.info("Triggering job restart");
            this.jobRestartSignal.restartJob();
        } else {
            prepareWorkerExecution(jobModel, newMetadata, jobMetadataChanges);
            // save components that depend on job model in order to manage lifecycle or access later
            this.currentDiagnosticsManager = diagnosticsManager;
            this.currentJobModelMonitors = Optional.of(jobModelMonitors);
            this.currentJobModel = Optional.of(jobModel);
            // lifecycle: start components
            this.coordinatorCommunication.start();
            this.jobCoordinatorListener.ifPresent(listener -> listener.onNewJobModel(this.processorId, jobModel));
            this.currentDiagnosticsManager.ifPresent(DiagnosticsManager::start);
            jobModelMonitors.start();
            this.jobPreparationComplete.set(true);
        }
    } catch (Exception e) {
        LOG.error("Error while running job coordinator; exiting", e);
        throw new SamzaException("Error while running job coordinator", e);
    }
}

Example 10 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestJobCoordinatorMetadataManager method testCheckForMetadataChangesNewDeployment.

@Test
public void testCheckForMetadataChangesNewDeployment() {
    JobCoordinatorMetadata previousMetadata = new JobCoordinatorMetadata(OLD_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    JobCoordinatorMetadata newMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, OLD_CONFIG_ID, OLD_JOB_MODEL_ID);
    Set<JobMetadataChange> metadataChanges = this.jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata);
    assertEquals("Metadata check should indicate new deployment", ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT), metadataChanges);
    assertEquals("New deployment should be 1 since Epoch ID changed", 1, this.jobCoordinatorMetadataManager.getMetrics().getNewDeployment().getValue().intValue());
    assertEquals("Application attempt count should be 0", 0, this.jobCoordinatorMetadataManager.getMetrics().getApplicationAttemptCount().getValue().intValue());
}