Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 51 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestHostAwareContainerAllocator method getConfig.

private static Config getConfig() {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            put("yarn.container.count", "1");
            put("systems.test-system.samza.factory", "org.apache.samza.job.yarn.MockSystemFactory");
            put("yarn.container.memory.mb", "512");
            put("yarn.package.path", "/foo");
            put("task.inputs", "test-system.test-stream");
            put("systems.test-system.samza.key.serde", "org.apache.samza.serializers.JsonSerde");
            put("systems.test-system.samza.msg.serde", "org.apache.samza.serializers.JsonSerde");
            put("yarn.container.retry.count", "1");
            put("yarn.container.retry.window.ms", "1999999999");
            put("yarn.samza.host-affinity.enabled", "true");
            put("yarn.container.request.timeout.ms", "3");
            put("yarn.allocator.sleep.ms", "1");
        }
    });
    Map<String, String> map = new HashMap<>();
    map.putAll(config);
    return new MapConfig(map);
}

Example 52 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestGroupByPartition method testBroadcastStreamsGroupedCorrectly.

@Test
public void testBroadcastStreamsGroupedCorrectly() {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put("task.broadcast.inputs", "SystemA.StreamA#0, SystemA.StreamB#1");
    Config config = new MapConfig(configMap);
    GroupByPartition grouper = new GroupByPartition(config);
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    Collections.addAll(allSSPs, aa0, aa1, aa2, ab1, ab2, ac0);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition0 = new HashSet<SystemStreamPartition>();
    // broadcast stream
    partition0.add(aa0);
    partition0.add(ac0);
    // broadcast stream
    partition0.add(ab1);
    expectedResult.put(new TaskName("Partition 0"), partition0);
    HashSet<SystemStreamPartition> partition1 = new HashSet<SystemStreamPartition>();
    partition1.add(aa1);
    // broadcast stream
    partition1.add(ab1);
    // broadcast stream
    partition1.add(aa0);
    expectedResult.put(new TaskName("Partition 1"), partition1);
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    partition2.add(aa2);
    partition2.add(ab2);
    // broadcast stream
    partition2.add(aa0);
    // broadcast stream
    partition2.add(ab1);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}

Example 53 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testCreateStreamApplicationWithTaskClass.

@Test
public void testCreateStreamApplicationWithTaskClass() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.TestStreamApplication");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    assertNotNull(streamApp);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "org.apache.samza.testUtils.TestAsyncStreamTask");
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.TestStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("should have failed with invalid config");
    } catch (ConfigException ce) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put("task.class", "no.such.class");
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.TestStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("should have failed with invalid config");
    } catch (ConfigException ce) {
    // expected
    }
}

Example 54 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestHdfsSystemConsumer method generateDefaultConfig.

private Config generateDefaultConfig() throws IOException {
    Map<String, String> properties = new HashMap<>();
    properties.put(String.format(HdfsConfig.CONSUMER_PARTITIONER_WHITELIST(), SYSTEM_NAME), ".*TestHdfsSystemConsumer.*avro");
    Path stagingDirectory = Files.createTempDirectory("staging");
    stagingDirectory.toFile().deleteOnExit();
    properties.put(HdfsConfig.STAGING_DIRECTORY(), stagingDirectory.toString());
    return new MapConfig(properties);
}

Example 55 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestHdfsSystemConsumer method testEmptyStagingDirectory.

/*
   * Ensure that empty staging directory will not break system admin,
   * but should fail system consumer
   */
@Test
public void testEmptyStagingDirectory() throws Exception {
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(HdfsConfig.CONSUMER_PARTITIONER_WHITELIST(), SYSTEM_NAME), ".*avro");
    Config config = new MapConfig(configMap);
    HdfsSystemFactory systemFactory = new HdfsSystemFactory();
    // create admin and do partitioning
    HdfsSystemAdmin systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config);
    String stream = WORKING_DIRECTORY;
    Set<String> streamNames = new HashSet<>();
    streamNames.add(stream);
    generateAvroDataFiles();
    Map<String, SystemStreamMetadata> streamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
    SystemStreamMetadata systemStreamMetadata = streamMetadataMap.get(stream);
    Assert.assertEquals(NUM_FILES, systemStreamMetadata.getSystemStreamPartitionMetadata().size());
    // create consumer and read from files
    HdfsSystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, new NoOpMetricsRegistry());
    Partition partition = new Partition(0);
    SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, stream, partition);
    try {
        systemConsumer.register(ssp, "0");
        Assert.fail("Empty staging directory should fail system consumer");
    } catch (UncheckedExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof SamzaException);
    }
}