Examples with JobCoordinatorMetadata org.apache.samza.job.JobCoordinatorMetadata used on opensource projects

Example 16 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestClusterBasedJobCoordinator method testGenerateAndUpdateJobCoordinatorMetadata.

@Test
public void testGenerateAndUpdateJobCoordinatorMetadata() {
    Config jobConfig = new MapConfig(configMap);
    when(CoordinatorStreamUtil.readConfigFromCoordinatorStream(anyObject())).thenReturn(jobConfig);
    ClusterBasedJobCoordinator clusterBasedJobCoordinator = spy(ClusterBasedJobCoordinatorRunner.createFromMetadataStore(jobConfig));
    JobCoordinatorMetadata previousMetadata = mock(JobCoordinatorMetadata.class);
    JobCoordinatorMetadata newMetadata = mock(JobCoordinatorMetadata.class);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = mock(JobCoordinatorMetadataManager.class);
    JobModel mockJobModel = mock(JobModel.class);
    when(jobCoordinatorMetadataManager.readJobCoordinatorMetadata()).thenReturn(previousMetadata);
    when(jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(any(), any())).thenReturn(newMetadata);
    when(jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata)).thenReturn(ImmutableSet.of());
    when(clusterBasedJobCoordinator.createJobCoordinatorMetadataManager()).thenReturn(jobCoordinatorMetadataManager);
    /*
     * Verify if there are no changes to metadata, the metadata changed flag remains false and no interactions
     * with job coordinator metadata manager
     */
    clusterBasedJobCoordinator.generateAndUpdateJobCoordinatorMetadata(mockJobModel);
    assertFalse("JC metadata changed should remain unchanged", clusterBasedJobCoordinator.isMetadataChangedAcrossAttempts());
    verify(jobCoordinatorMetadataManager, times(0)).writeJobCoordinatorMetadata(any());
    /*
     * Verify if there are changes to metadata, we persist the new metadata & update the metadata changed flag
     */
    when(jobCoordinatorMetadataManager.checkForMetadataChanges(newMetadata, previousMetadata)).thenReturn(ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT));
    clusterBasedJobCoordinator.generateAndUpdateJobCoordinatorMetadata(mockJobModel);
    assertTrue("JC metadata changed should be true", clusterBasedJobCoordinator.isMetadataChangedAcrossAttempts());
    verify(jobCoordinatorMetadataManager, times(1)).writeJobCoordinatorMetadata(newMetadata);
}

Example 17 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class JobCoordinatorMetadataManager method readJobCoordinatorMetadata.

/**
 * Reads the {@link JobCoordinatorMetadata} from the metadata store. It fetches the metadata
 * associated with cluster type specified at the creation of the manager.
 *
 * @return job coordinator metadata
 */
public JobCoordinatorMetadata readJobCoordinatorMetadata() {
    JobCoordinatorMetadata metadata = null;
    for (Map.Entry<String, byte[]> entry : metadataStore.all().entrySet()) {
        if (clusterType.name().equals(entry.getKey())) {
            try {
                String metadataString = valueSerde.fromBytes(entry.getValue());
                metadata = metadataMapper.readValue(metadataString, JobCoordinatorMetadata.class);
                break;
            } catch (Exception e) {
                metrics.incrementMetadataReadFailedCount();
                LOG.error("Failed to read job coordinator metadata due to ", e);
            }
        }
    }
    LOG.info("Fetched the job coordinator metadata for cluster {} as {}.", clusterType, metadata);
    return metadata;
}

Example 18 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class JobCoordinatorMetadataManager method generateJobCoordinatorMetadata.

/**
 * Generates {@link JobCoordinatorMetadata}.
 *
 * Epoch ID - It is generated by {@link #fetchEpochIdForJobCoordinator()}. Refer to the javadocs for more
 * details on how it is generated and the properties of the identifier.
 *
 * Config ID - A unique and reproducible identifier that is generated based on the input {@link Config}. It uses
 * a {@link Funnel} to use a subset of the input configuration to generate the identifier and as long as the subset
 * of the configuration remains same, the identifier is guaranteed to be same. For the list of config prefixes used
 * by the funnel refer to {@link ConfigHashFunnel}
 *
 * JobModel ID - A unique and reproducible identifier that is generated based on the input {@link JobModel}. It only
 * uses the {@link org.apache.samza.job.model.ContainerModel} within the {@linkplain JobModel} for generation. We
 * serialize the data into bytes and use those bytes to compute the identifier.
 *
 * In case of YARN, the epoch identifier is extracted from the application attempt and translates to applicationId
 * e.g. 1606797336059_0010
 * Both config and job model identifiers should a 32 bit integer.
 *
 * @param jobModel job model used for generating the metadata
 *
 * @return the metadata for the job coordinator
 */
public JobCoordinatorMetadata generateJobCoordinatorMetadata(JobModel jobModel, Config config) {
    try {
        int jobModelId = Hashing.crc32c().hashBytes(SamzaObjectMapper.getObjectMapper().writeValueAsBytes(jobModel.getContainers())).asInt();
        int configId = Hashing.crc32().hashObject(config, new ConfigHashFunnel()).asInt();
        LOG.info("Generated job model id {} and config id {}", jobModelId, configId);
        return new JobCoordinatorMetadata(fetchEpochIdForJobCoordinator(), String.valueOf(configId), String.valueOf(jobModelId));
    } catch (Exception e) {
        metrics.incrementMetadataGenerationFailedCount();
        LOG.error("Failed to generate metadata for the current attempt due to ", e);
        throw new SamzaException("Failed to generate the metadata for the current attempt due to ", e);
    }
}

Example 19 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestStaticResourceJobCoordinator method testStreamRegexChange.

@Test
public void testStreamRegexChange() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = mock(StreamRegexMonitor.class);
    ArgumentCaptor<StreamRegexMonitor.Callback> callbackArgumentCaptor = ArgumentCaptor.forClass(StreamRegexMonitor.Callback.class);
    when(this.streamRegexMonitorFactory.build(eq(jobModel), eq(jobModelConfig), callbackArgumentCaptor.capture())).thenReturn(Optional.of(streamRegexMonitor));
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    // call the callback from the monitor
    callbackArgumentCaptor.getValue().onInputStreamsChanged(ImmutableSet.of(SYSTEM_STREAM), ImmutableSet.of(SYSTEM_STREAM, new SystemStream("system", "stream1")), ImmutableMap.of("system", Pattern.compile("stream.*")));
    verify(this.jobRestartSignal).restartJob();
}

Example 20 with JobCoordinatorMetadata

use of org.apache.samza.job.JobCoordinatorMetadata in project samza by apache.

the class TestStaticResourceJobCoordinator method testNewDeploymentNewJobModel.

@Test
public void testNewDeploymentNewJobModel() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}