Examples with MetadataResourceUtil org.apache.samza.coordinator.MetadataResourceUtil used on opensource projects

Example 6 with MetadataResourceUtil

use of org.apache.samza.coordinator.MetadataResourceUtil in project samza by apache.

the class PassthroughJobCoordinator method start.

@Override
public void start() {
    // No-op
    JobModel jobModel = null;
    try {
        jobModel = getJobModel();
        // TODO metrics registry has been null here for a while; is it safe?
        MetadataResourceUtil metadataResourceUtil = new MetadataResourceUtil(jobModel, null, config);
        metadataResourceUtil.createResources();
    } catch (Exception e) {
        LOGGER.error("Exception while trying to getJobModel.", e);
        if (coordinatorListener != null) {
            coordinatorListener.onCoordinatorFailure(e);
        }
    }
    if (jobModel != null && jobModel.getContainers().containsKey(processorId)) {
        if (coordinatorListener != null) {
            coordinatorListener.onJobModelExpired();
            coordinatorListener.onNewJobModel(processorId, jobModel);
        }
    } else {
        LOGGER.info("JobModel: {} does not contain processorId: {}. Stopping the JobCoordinator", jobModel, processorId);
        stop();
    }
}

Example 7 with MetadataResourceUtil

use of org.apache.samza.coordinator.MetadataResourceUtil in project samza by apache.

the class TestZkJobCoordinator method testLoadMetadataResources.

@Test
public void testLoadMetadataResources() throws IOException {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(jobModel);
    StartpointManager mockStartpointManager = Mockito.mock(StartpointManager.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, config, new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    doReturn(mockStartpointManager).when(zkJobCoordinator).createStartpointManager();
    MetadataResourceUtil mockMetadataResourceUtil = mock(MetadataResourceUtil.class);
    doReturn(mockMetadataResourceUtil).when(zkJobCoordinator).createMetadataResourceUtil(any(), any(Config.class));
    verifyZeroInteractions(mockStartpointManager);
    zkJobCoordinator.loadMetadataResources(jobModel);
    verify(mockMetadataResourceUtil).createResources();
    verify(mockStartpointManager).start();
    verify(mockStartpointManager).fanOut(any());
    verify(mockStartpointManager).stop();
}

Example 8 with MetadataResourceUtil

use of org.apache.samza.coordinator.MetadataResourceUtil in project samza by apache.

the class TestStaticResourceJobCoordinator method metadataResourceUtil.

private MetadataResourceUtil metadataResourceUtil(JobModel jobModel) {
    MetadataResourceUtil metadataResourceUtil = mock(MetadataResourceUtil.class);
    doReturn(metadataResourceUtil).when(this.staticResourceJobCoordinator).metadataResourceUtil(jobModel);
    return metadataResourceUtil;
}

Example 9 with MetadataResourceUtil

use of org.apache.samza.coordinator.MetadataResourceUtil in project samza by apache.

the class TestStaticResourceJobCoordinator method testStreamRegexChange.

@Test
public void testStreamRegexChange() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = mock(StreamRegexMonitor.class);
    ArgumentCaptor<StreamRegexMonitor.Callback> callbackArgumentCaptor = ArgumentCaptor.forClass(StreamRegexMonitor.Callback.class);
    when(this.streamRegexMonitorFactory.build(eq(jobModel), eq(jobModelConfig), callbackArgumentCaptor.capture())).thenReturn(Optional.of(streamRegexMonitor));
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    // call the callback from the monitor
    callbackArgumentCaptor.getValue().onInputStreamsChanged(ImmutableSet.of(SYSTEM_STREAM), ImmutableSet.of(SYSTEM_STREAM, new SystemStream("system", "stream1")), ImmutableMap.of("system", Pattern.compile("stream.*")));
    verify(this.jobRestartSignal).restartJob();
}

Example 10 with MetadataResourceUtil

use of org.apache.samza.coordinator.MetadataResourceUtil in project samza by apache.

the class TestStaticResourceJobCoordinator method testNewDeploymentNewJobModel.

@Test
public void testNewDeploymentNewJobModel() throws IOException {
    Config jobModelConfig = mock(Config.class);
    JobModel jobModel = setupJobModel(jobModelConfig);
    StreamPartitionCountMonitor streamPartitionCountMonitor = setupStreamPartitionCountMonitor(jobModelConfig);
    StreamRegexMonitor streamRegexMonitor = setupStreamRegexMonitor(jobModel, jobModelConfig);
    JobCoordinatorMetadata newMetadata = setupJobCoordinatorMetadata(jobModel, jobModelConfig, ImmutableSet.of(JobMetadataChange.NEW_DEPLOYMENT, JobMetadataChange.JOB_MODEL), true);
    setUpDiagnosticsManager(jobModel);
    MetadataResourceUtil metadataResourceUtil = metadataResourceUtil(jobModel);
    this.staticResourceJobCoordinator.start();
    assertEquals(jobModel, this.staticResourceJobCoordinator.getJobModel());
    verifyStartLifecycle();
    verify(this.staticResourceJobCoordinator).doSetLoggingContextConfig(jobModelConfig);
    verify(this.diagnosticsManager).start();
    verifyPrepareWorkerExecutionAndMonitor(jobModel, metadataResourceUtil, streamPartitionCountMonitor, streamRegexMonitor, newMetadata, SINGLE_SSP_FANOUT);
    verify(this.jobCoordinatorListener).onNewJobModel(PROCESSOR_ID, jobModel);
}