Search in sources :

Example 1 with StreamApplication

use of org.apache.samza.application.StreamApplication in project samza by apache.

the class TestWindowOperator method testTumblingWindowsAccumulatingMode.

@Test
public void testTumblingWindowsAccumulatingMode() throws Exception {
    StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
    List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
    TestClock testClock = new TestClock();
    StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
    task.init(config, taskContext);
    MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
    integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
    testClock.advanceTime(Duration.ofSeconds(1));
    task.window(messageCollector, taskCoordinator);
    Assert.assertEquals(windowPanes.size(), 7);
    Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
    Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
    Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
    Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);
    Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
    Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4);
    Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
    Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4);
}
Also used : TestClock(org.apache.samza.testUtils.TestClock) StreamOperatorTask(org.apache.samza.task.StreamOperatorTask) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Function(java.util.function.Function) WindowPane(org.apache.samza.operators.windows.WindowPane) ArrayList(java.util.ArrayList) Trigger(org.apache.samza.operators.triggers.Trigger) ImmutableList(com.google.common.collect.ImmutableList) AccumulationMode(org.apache.samza.operators.windows.AccumulationMode) MessageCollector(org.apache.samza.task.MessageCollector) FiringType(org.apache.samza.operators.triggers.FiringType) SystemStream(org.apache.samza.system.SystemStream) Duration(java.time.Duration) Before(org.junit.Before) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) TaskContext(org.apache.samza.task.TaskContext) ImmutableSet(com.google.common.collect.ImmutableSet) Windows(org.apache.samza.operators.windows.Windows) IncomingMessageEnvelope(org.apache.samza.system.IncomingMessageEnvelope) Collection(java.util.Collection) Partition(org.apache.samza.Partition) StreamSpec(org.apache.samza.system.StreamSpec) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) TaskCoordinator(org.apache.samza.task.TaskCoordinator) Triggers(org.apache.samza.operators.triggers.Triggers) List(java.util.List) TestClock(org.apache.samza.testUtils.TestClock) Assert(junit.framework.Assert) OutgoingMessageEnvelope(org.apache.samza.system.OutgoingMessageEnvelope) Config(org.apache.samza.config.Config) StreamApplication(org.apache.samza.application.StreamApplication) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Mockito.mock(org.mockito.Mockito.mock) StreamApplication(org.apache.samza.application.StreamApplication) StreamOperatorTask(org.apache.samza.task.StreamOperatorTask) MessageCollector(org.apache.samza.task.MessageCollector) ArrayList(java.util.ArrayList) Collection(java.util.Collection) WindowPane(org.apache.samza.operators.windows.WindowPane) Test(org.junit.Test)

Example 2 with StreamApplication

use of org.apache.samza.application.StreamApplication in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreation.

@Test
public void testStreamCreation() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}
Also used : StreamSpec(org.apache.samza.system.StreamSpec) HashMap(java.util.HashMap) StreamApplication(org.apache.samza.application.StreamApplication) ExecutionPlanner(org.apache.samza.execution.ExecutionPlanner) Matchers.anyString(org.mockito.Matchers.anyString) JobConfig(org.apache.samza.config.JobConfig) Field(java.lang.reflect.Field) ExecutionPlan(org.apache.samza.execution.ExecutionPlan) StreamManager(org.apache.samza.execution.StreamManager) List(java.util.List) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 3 with StreamApplication

use of org.apache.samza.application.StreamApplication in project samza by apache.

the class TestLocalApplicationRunner method testRunFailure.

@Test
public void testRunFailure() throws Exception {
    final Map<String, String> config = new HashMap<>();
    config.put(ApplicationConfig.PROCESSOR_ID, "0");
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.singletonList(new JobConfig(new MapConfig(config)));
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.emptyList();
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    Throwable t = new Throwable("test failure");
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessorLifecycleListener> captor = ArgumentCaptor.forClass(StreamProcessorLifecycleListener.class);
    doAnswer(i -> {
        StreamProcessorLifecycleListener listener = captor.getValue();
        listener.onFailure(t);
        return null;
    }).when(sp).start();
    LocalApplicationRunner spy = spy(runner);
    doReturn(sp).when(spy).createStreamProcessor(anyObject(), anyObject(), captor.capture());
    try {
        spy.run(app);
    } catch (Throwable th) {
        assertNotNull(th);
    }
    assertEquals(spy.status(app), ApplicationStatus.UnsuccessfulFinish);
}
Also used : StreamSpec(org.apache.samza.system.StreamSpec) StreamProcessor(org.apache.samza.processor.StreamProcessor) HashMap(java.util.HashMap) StreamApplication(org.apache.samza.application.StreamApplication) ExecutionPlanner(org.apache.samza.execution.ExecutionPlanner) Matchers.anyString(org.mockito.Matchers.anyString) StreamProcessorLifecycleListener(org.apache.samza.processor.StreamProcessorLifecycleListener) JobConfig(org.apache.samza.config.JobConfig) Field(java.lang.reflect.Field) ExecutionPlan(org.apache.samza.execution.ExecutionPlan) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 4 with StreamApplication

use of org.apache.samza.application.StreamApplication in project samza by apache.

the class TestTaskFactoryUtil method testStreamTaskClassWithInvalidStreamApplication.

@Test
public void testStreamTaskClassWithInvalidStreamApplication() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException ce) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestStreamTask");
            this.put(ApplicationConfig.APP_CLASS, "");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof StreamTaskFactory);
    assertTrue(((StreamTaskFactory) retFactory).createInstance() instanceof TestStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "");
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no class not found");
    } catch (ConfigException cne) {
    // expected
    }
}
Also used : HashMap(java.util.HashMap) ApplicationConfig(org.apache.samza.config.ApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) TestStreamTask(org.apache.samza.testUtils.TestStreamTask) StreamApplication(org.apache.samza.application.StreamApplication) ConfigException(org.apache.samza.config.ConfigException) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 5 with StreamApplication

use of org.apache.samza.application.StreamApplication in project samza by apache.

the class TestTaskFactoryUtil method testAsyncStreamTaskWithInvalidStreamGraphBuilder.

@Test
public void testAsyncStreamTaskWithInvalidStreamGraphBuilder() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException cfe) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
            this.put(ApplicationConfig.APP_CLASS, "");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof TestAsyncStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
            this.put(ApplicationConfig.APP_CLASS, null);
        }
    });
    streamApp = TaskFactoryUtil.createStreamApplication(config);
    retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof TestAsyncStreamTask);
}
Also used : HashMap(java.util.HashMap) ApplicationConfig(org.apache.samza.config.ApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) StreamApplication(org.apache.samza.application.StreamApplication) ConfigException(org.apache.samza.config.ConfigException) MapConfig(org.apache.samza.config.MapConfig) TestAsyncStreamTask(org.apache.samza.testUtils.TestAsyncStreamTask) Test(org.junit.Test)

Aggregations

StreamApplication (org.apache.samza.application.StreamApplication)19 Test (org.junit.Test)16 Config (org.apache.samza.config.Config)14 StreamSpec (org.apache.samza.system.StreamSpec)12 List (java.util.List)10 ImmutableList (com.google.common.collect.ImmutableList)8 ImmutableSet (com.google.common.collect.ImmutableSet)8 Duration (java.time.Duration)8 ArrayList (java.util.ArrayList)8 Collection (java.util.Collection)8 HashMap (java.util.HashMap)8 Function (java.util.function.Function)8 Assert (junit.framework.Assert)8 Partition (org.apache.samza.Partition)8 MapConfig (org.apache.samza.config.MapConfig)8 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)8 FiringType (org.apache.samza.operators.triggers.FiringType)8 Trigger (org.apache.samza.operators.triggers.Trigger)8 Triggers (org.apache.samza.operators.triggers.Triggers)8 AccumulationMode (org.apache.samza.operators.windows.AccumulationMode)8