Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 6 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testStreamTaskClassWithInvalidStreamApplication.

@Test
public void testStreamTaskClassWithInvalidStreamApplication() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException ce) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestStreamTask");
            this.put(ApplicationConfig.APP_CLASS, "");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof StreamTaskFactory);
    assertTrue(((StreamTaskFactory) retFactory).createInstance() instanceof TestStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "");
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no class not found");
    } catch (ConfigException cne) {
    // expected
    }
}

Example 7 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testStreamTaskClass.

@Test
public void testStreamTaskClass() {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestStreamTask");
        }
    });
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, null, null);
    assertTrue(retFactory instanceof StreamTaskFactory);
    assertTrue(((StreamTaskFactory) retFactory).createInstance() instanceof TestStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "no.such.class");
        }
    });
    try {
        TaskFactoryUtil.createTaskFactory(config, null, null);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException cfe) {
    // expected
    }
}

Example 8 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testAsyncStreamTask.

@Test
public void testAsyncStreamTask() {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
        }
    });
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, null, null);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof TestAsyncStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "no.such.class");
        }
    });
    try {
        TaskFactoryUtil.createTaskFactory(config, null, null);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException cfe) {
    // expected
    }
}

Example 9 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testAsyncStreamTaskWithInvalidStreamGraphBuilder.

@Test
public void testAsyncStreamTaskWithInvalidStreamGraphBuilder() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException cfe) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
            this.put(ApplicationConfig.APP_CLASS, "");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof TestAsyncStreamTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
            this.put(ApplicationConfig.APP_CLASS, null);
        }
    });
    streamApp = TaskFactoryUtil.createStreamApplication(config);
    retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof AsyncStreamTaskFactory);
    assertTrue(((AsyncStreamTaskFactory) retFactory).createInstance() instanceof TestAsyncStreamTask);
}

Example 10 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestJobGraphJsonGenerator method test.

@Test
public void test() throws Exception {
    /**
     * the graph looks like the following. number of partitions in parentheses. quotes indicate expected value.
     *
     *                               input1 (64) -> map -> join -> output1 (8)
     *                                                       |
     *          input2 (16) -> partitionBy ("64") -> filter -|
     *                                                       |
     * input3 (32) -> filter -> partitionBy ("64") -> map -> join -> output2 (16)
     *
     */
    Map<String, String> configMap = new HashMap<>();
    configMap.put(JobConfig.JOB_NAME(), "test-app");
    configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system");
    Config config = new MapConfig(configMap);
    StreamSpec input1 = new StreamSpec("input1", "input1", "system1");
    StreamSpec input2 = new StreamSpec("input2", "input2", "system2");
    StreamSpec input3 = new StreamSpec("input3", "input3", "system2");
    StreamSpec output1 = new StreamSpec("output1", "output1", "system1");
    StreamSpec output2 = new StreamSpec("output2", "output2", "system2");
    ApplicationRunner runner = mock(ApplicationRunner.class);
    when(runner.getStreamSpec("input1")).thenReturn(input1);
    when(runner.getStreamSpec("input2")).thenReturn(input2);
    when(runner.getStreamSpec("input3")).thenReturn(input3);
    when(runner.getStreamSpec("output1")).thenReturn(output1);
    when(runner.getStreamSpec("output2")).thenReturn(output2);
    // intermediate streams used in tests
    when(runner.getStreamSpec("test-app-1-partition_by-0")).thenReturn(new StreamSpec("test-app-1-partition_by-0", "test-app-1-partition_by-0", "default-system"));
    when(runner.getStreamSpec("test-app-1-partition_by-1")).thenReturn(new StreamSpec("test-app-1-partition_by-1", "test-app-1-partition_by-1", "default-system"));
    when(runner.getStreamSpec("test-app-1-partition_by-4")).thenReturn(new StreamSpec("test-app-1-partition_by-4", "test-app-1-partition_by-4", "default-system"));
    // set up external partition count
    Map<String, Integer> system1Map = new HashMap<>();
    system1Map.put("input1", 64);
    system1Map.put("output1", 8);
    Map<String, Integer> system2Map = new HashMap<>();
    system2Map.put("input2", 16);
    system2Map.put("input3", 32);
    system2Map.put("output2", 16);
    Map<String, SystemAdmin> systemAdmins = new HashMap<>();
    SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
    SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
    systemAdmins.put("system1", systemAdmin1);
    systemAdmins.put("system2", systemAdmin2);
    StreamManager streamManager = new StreamManager(systemAdmins);
    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
    BiFunction mockBuilder = mock(BiFunction.class);
    MessageStream m1 = streamGraph.getInputStream("input1", mockBuilder).map(m -> m);
    MessageStream m2 = streamGraph.getInputStream("input2", mockBuilder).partitionBy(m -> "haha").filter(m -> true);
    MessageStream m3 = streamGraph.getInputStream("input3", mockBuilder).filter(m -> true).partitionBy(m -> "hehe").map(m -> m);
    Function mockFn = mock(Function.class);
    OutputStream<Object, Object, Object> outputStream1 = streamGraph.getOutputStream("output1", mockFn, mockFn);
    OutputStream<Object, Object, Object> outputStream2 = streamGraph.getOutputStream("output2", mockFn, mockFn);
    m1.join(m2, mock(JoinFunction.class), Duration.ofHours(2)).sendTo(outputStream1);
    m2.sink((message, collector, coordinator) -> {
    });
    m3.join(m2, mock(JoinFunction.class), Duration.ofHours(1)).sendTo(outputStream2);
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    ExecutionPlan plan = planner.plan(streamGraph);
    String json = plan.getPlanAsJson();
    System.out.println(json);
    // deserialize
    ObjectMapper mapper = new ObjectMapper();
    JobGraphJsonGenerator.JobGraphJson nodes = mapper.readValue(json, JobGraphJsonGenerator.JobGraphJson.class);
    assertTrue(nodes.jobs.get(0).operatorGraph.inputStreams.size() == 5);
    assertTrue(nodes.jobs.get(0).operatorGraph.operators.size() == 13);
    assertTrue(nodes.sourceStreams.size() == 3);
    assertTrue(nodes.sinkStreams.size() == 2);
    assertTrue(nodes.intermediateStreams.size() == 2);
}