Search in sources :

Example 1 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testKill.

@Test
public void testKill() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        return null;
    }).when(sp).start();
    doAnswer(new Answer() {

        private int count = 0;

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            if (++count == 1) {
                ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
                listener.afterStop();
                return null;
            }
            return null;
        }
    }).when(sp).stop();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.kill();
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore, atLeastOnce()).close();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}
Also used : LegacyTaskApplication(org.apache.samza.application.LegacyTaskApplication) StreamProcessor(org.apache.samza.processor.StreamProcessor) ApplicationDescriptorUtil(org.apache.samza.application.descriptors.ApplicationDescriptorUtil) Matchers.eq(org.mockito.Matchers.eq) Duration(java.time.Duration) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) SamzaApplication(org.apache.samza.application.SamzaApplication) InMemoryMetadataStoreFactory(org.apache.samza.metadatastore.InMemoryMetadataStoreFactory) ExternalContext(org.apache.samza.context.ExternalContext) InMemoryMetadataStore(org.apache.samza.metadatastore.InMemoryMetadataStore) MapConfig(org.apache.samza.config.MapConfig) Mockito.doReturn(org.mockito.Mockito.doReturn) ClusterMembership(org.apache.samza.coordinator.ClusterMembership) ImmutableMap(com.google.common.collect.ImmutableMap) CoordinationUtils(org.apache.samza.coordinator.CoordinationUtils) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) CoordinatorStreamMetadataStoreFactory(org.apache.samza.coordinator.metadatastore.CoordinatorStreamMetadataStoreFactory) PassthroughJobCoordinatorFactory(org.apache.samza.standalone.PassthroughJobCoordinatorFactory) DistributedLock(org.apache.samza.coordinator.DistributedLock) Matchers.any(org.mockito.Matchers.any) CoordinationConstants(org.apache.samza.coordinator.CoordinationConstants) MetadataStoreFactory(org.apache.samza.metadatastore.MetadataStoreFactory) IdentityStreamTask(org.apache.samza.task.IdentityStreamTask) Assert.assertFalse(org.junit.Assert.assertFalse) ZkMetadataStoreFactory(org.apache.samza.zk.ZkMetadataStoreFactory) Optional(java.util.Optional) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) StreamApplication(org.apache.samza.application.StreamApplication) ApplicationStatus(org.apache.samza.job.ApplicationStatus) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) ConfigException(org.apache.samza.config.ConfigException) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) RunWith(org.junit.runner.RunWith) JobConfig(org.apache.samza.config.JobConfig) HashMap(java.util.HashMap) Mockito.spy(org.mockito.Mockito.spy) Matchers.anyString(org.mockito.Matchers.anyString) Answer(org.mockito.stubbing.Answer) MetadataStore(org.apache.samza.metadatastore.MetadataStore) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Matchers.anyObject(org.mockito.Matchers.anyObject) ApplicationConfig(org.apache.samza.config.ApplicationConfig) ApplicationDescriptor(org.apache.samza.application.descriptors.ApplicationDescriptor) PowerMockRunner(org.powermock.modules.junit4.PowerMockRunner) PowerMockito(org.powermock.api.mockito.PowerMockito) Before(org.junit.Before) Assert.assertNotNull(org.junit.Assert.assertNotNull) ZkMetadataStore(org.apache.samza.zk.ZkMetadataStore) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) Mockito(org.mockito.Mockito) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) ApplicationDescriptorImpl(org.apache.samza.application.descriptors.ApplicationDescriptorImpl) LocalJobPlanner(org.apache.samza.execution.LocalJobPlanner) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) StreamProcessor(org.apache.samza.processor.StreamProcessor) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) ApplicationConfig(org.apache.samza.config.ApplicationConfig) Matchers.anyString(org.mockito.Matchers.anyString) JobConfig(org.apache.samza.config.JobConfig) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Answer(org.mockito.stubbing.Answer) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ExternalContext(org.apache.samza.context.ExternalContext) Matchers.anyObject(org.mockito.Matchers.anyObject) MapConfig(org.apache.samza.config.MapConfig) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 2 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTaskWithoutExternalContext.

@Test
public void testRunStreamTaskWithoutExternalContext() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.empty()), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run();
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}
Also used : StreamProcessor(org.apache.samza.processor.StreamProcessor) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) ApplicationConfig(org.apache.samza.config.ApplicationConfig) LegacyTaskApplication(org.apache.samza.application.LegacyTaskApplication) Matchers.anyString(org.mockito.Matchers.anyString) MapConfig(org.apache.samza.config.MapConfig) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 3 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTask.

@Test
public void testRunStreamTask() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run(externalContext);
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}
Also used : StreamProcessor(org.apache.samza.processor.StreamProcessor) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) ApplicationConfig(org.apache.samza.config.ApplicationConfig) LegacyTaskApplication(org.apache.samza.application.LegacyTaskApplication) Matchers.anyString(org.mockito.Matchers.anyString) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) ExternalContext(org.apache.samza.context.ExternalContext) MapConfig(org.apache.samza.config.MapConfig) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method prepareTestForRunId.

private void prepareTestForRunId() throws Exception {
    coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock lock = mock(DistributedLock.class);
    when(lock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(lock);
    clusterMembership = mock(ClusterMembership.class);
    when(clusterMembership.getNumberOfProcessors()).thenReturn(1);
    when(coordinationUtils.getClusterMembership()).thenReturn(clusterMembership);
    metadataStore = mock(ZkMetadataStore.class);
    when(metadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(metadataStore);
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(mockApp, config);
    runner = spy(new LocalApplicationRunner(mockApp, config, coordinationUtils));
    localPlanner = spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
    doReturn(localPlanner).when(runner).getPlanner();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), anyObject(), anyObject(), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
}
Also used : DistributedLock(org.apache.samza.coordinator.DistributedLock) StreamProcessor(org.apache.samza.processor.StreamProcessor) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) ClusterMembership(org.apache.samza.coordinator.ClusterMembership) MapConfig(org.apache.samza.config.MapConfig) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) ApplicationConfig(org.apache.samza.config.ApplicationConfig) LocalJobPlanner(org.apache.samza.execution.LocalJobPlanner) ZkMetadataStore(org.apache.samza.zk.ZkMetadataStore) CoordinationUtils(org.apache.samza.coordinator.CoordinationUtils)

Example 5 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testRunFailure.

@Test
public void testRunFailure() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.PROCESSOR_ID, "0");
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        throw new Exception("test failure");
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    try {
        runner.run(externalContext);
        runner.waitForFinish();
    } catch (Throwable th) {
        assertNotNull(th);
    }
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore, never()).close();
    assertEquals(runner.status(), ApplicationStatus.UnsuccessfulFinish);
}
Also used : LegacyTaskApplication(org.apache.samza.application.LegacyTaskApplication) StreamProcessor(org.apache.samza.processor.StreamProcessor) ApplicationDescriptorUtil(org.apache.samza.application.descriptors.ApplicationDescriptorUtil) Matchers.eq(org.mockito.Matchers.eq) Duration(java.time.Duration) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) SamzaApplication(org.apache.samza.application.SamzaApplication) InMemoryMetadataStoreFactory(org.apache.samza.metadatastore.InMemoryMetadataStoreFactory) ExternalContext(org.apache.samza.context.ExternalContext) InMemoryMetadataStore(org.apache.samza.metadatastore.InMemoryMetadataStore) MapConfig(org.apache.samza.config.MapConfig) Mockito.doReturn(org.mockito.Mockito.doReturn) ClusterMembership(org.apache.samza.coordinator.ClusterMembership) ImmutableMap(com.google.common.collect.ImmutableMap) CoordinationUtils(org.apache.samza.coordinator.CoordinationUtils) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) CoordinatorStreamMetadataStoreFactory(org.apache.samza.coordinator.metadatastore.CoordinatorStreamMetadataStoreFactory) PassthroughJobCoordinatorFactory(org.apache.samza.standalone.PassthroughJobCoordinatorFactory) DistributedLock(org.apache.samza.coordinator.DistributedLock) Matchers.any(org.mockito.Matchers.any) CoordinationConstants(org.apache.samza.coordinator.CoordinationConstants) MetadataStoreFactory(org.apache.samza.metadatastore.MetadataStoreFactory) IdentityStreamTask(org.apache.samza.task.IdentityStreamTask) Assert.assertFalse(org.junit.Assert.assertFalse) ZkMetadataStoreFactory(org.apache.samza.zk.ZkMetadataStoreFactory) Optional(java.util.Optional) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) StreamApplication(org.apache.samza.application.StreamApplication) ApplicationStatus(org.apache.samza.job.ApplicationStatus) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) ConfigException(org.apache.samza.config.ConfigException) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) RunWith(org.junit.runner.RunWith) JobConfig(org.apache.samza.config.JobConfig) HashMap(java.util.HashMap) Mockito.spy(org.mockito.Mockito.spy) Matchers.anyString(org.mockito.Matchers.anyString) Answer(org.mockito.stubbing.Answer) MetadataStore(org.apache.samza.metadatastore.MetadataStore) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Matchers.anyObject(org.mockito.Matchers.anyObject) ApplicationConfig(org.apache.samza.config.ApplicationConfig) ApplicationDescriptor(org.apache.samza.application.descriptors.ApplicationDescriptor) PowerMockRunner(org.powermock.modules.junit4.PowerMockRunner) PowerMockito(org.powermock.api.mockito.PowerMockito) Before(org.junit.Before) Assert.assertNotNull(org.junit.Assert.assertNotNull) ZkMetadataStore(org.apache.samza.zk.ZkMetadataStore) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) Mockito(org.mockito.Mockito) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) ApplicationDescriptorImpl(org.apache.samza.application.descriptors.ApplicationDescriptorImpl) LocalJobPlanner(org.apache.samza.execution.LocalJobPlanner) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) StreamProcessor(org.apache.samza.processor.StreamProcessor) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) ApplicationConfig(org.apache.samza.config.ApplicationConfig) Matchers.anyString(org.mockito.Matchers.anyString) JobConfig(org.apache.samza.config.JobConfig) ConfigException(org.apache.samza.config.ConfigException) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) ExternalContext(org.apache.samza.context.ExternalContext) MapConfig(org.apache.samza.config.MapConfig) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

CoordinatorStreamStore (org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore)24 Config (org.apache.samza.config.Config)13 JobConfig (org.apache.samza.config.JobConfig)12 MapConfig (org.apache.samza.config.MapConfig)12 Test (org.junit.Test)12 JobCoordinatorConfig (org.apache.samza.config.JobCoordinatorConfig)10 ApplicationConfig (org.apache.samza.config.ApplicationConfig)8 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)8 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)8 HashMap (java.util.HashMap)7 Before (org.junit.Before)7 SamzaException (org.apache.samza.SamzaException)6 ImmutableMap (com.google.common.collect.ImmutableMap)5 Map (java.util.Map)5 StreamProcessor (org.apache.samza.processor.StreamProcessor)5 SystemAdmins (org.apache.samza.system.SystemAdmins)5 Optional (java.util.Optional)4 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)4 CountDownLatch (java.util.concurrent.CountDownLatch)4 LegacyTaskApplication (org.apache.samza.application.LegacyTaskApplication)4