Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 61 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestHdfsSystemConsumer method generateDefaultConfig.

private Config generateDefaultConfig() throws IOException {
    Map<String, String> properties = new HashMap<>();
    properties.put(String.format(HdfsConfig.CONSUMER_PARTITIONER_WHITELIST(), SYSTEM_NAME), ".*TestHdfsSystemConsumer.*avro");
    Path stagingDirectory = Files.createTempDirectory("staging");
    stagingDirectory.toFile().deleteOnExit();
    properties.put(HdfsConfig.STAGING_DIRECTORY(), stagingDirectory.toString());
    return new MapConfig(properties);
}

Example 62 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestHdfsSystemConsumer method testEmptyStagingDirectory.

/*
   * Ensure that empty staging directory will not break system admin,
   * but should fail system consumer
   */
@Test
public void testEmptyStagingDirectory() throws Exception {
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(HdfsConfig.CONSUMER_PARTITIONER_WHITELIST(), SYSTEM_NAME), ".*avro");
    Config config = new MapConfig(configMap);
    HdfsSystemFactory systemFactory = new HdfsSystemFactory();
    // create admin and do partitioning
    HdfsSystemAdmin systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config);
    String stream = WORKING_DIRECTORY;
    Set<String> streamNames = new HashSet<>();
    streamNames.add(stream);
    generateAvroDataFiles();
    Map<String, SystemStreamMetadata> streamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
    SystemStreamMetadata systemStreamMetadata = streamMetadataMap.get(stream);
    Assert.assertEquals(NUM_FILES, systemStreamMetadata.getSystemStreamPartitionMetadata().size());
    // create consumer and read from files
    HdfsSystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, new NoOpMetricsRegistry());
    Partition partition = new Partition(0);
    SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, stream, partition);
    try {
        systemConsumer.register(ssp, "0");
        Assert.fail("Empty staging directory should fail system consumer");
    } catch (UncheckedExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof SamzaException);
    }
}

Example 63 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestTaskFactoryUtil method testCreateStreamApplication.

@Test
public void testCreateStreamApplication() throws Exception {
    Config config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.TestStreamApplication");
        }
    });
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    assertNotNull(streamApp);
    Object retFactory = TaskFactoryUtil.createTaskFactory(config, streamApp, mockRunner);
    assertTrue(retFactory instanceof StreamTaskFactory);
    assertTrue(((StreamTaskFactory) retFactory).createInstance() instanceof StreamOperatorTask);
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "org.apache.samza.testUtils.InvalidStreamApplication");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException ce) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "no.such.class");
        }
    });
    try {
        TaskFactoryUtil.createStreamApplication(config);
        fail("Should have failed w/ no.such.class");
    } catch (ConfigException ce) {
    // expected
    }
    config = new MapConfig(new HashMap<String, String>() {

        {
            this.put(ApplicationConfig.APP_CLASS, "");
        }
    });
    streamApp = TaskFactoryUtil.createStreamApplication(config);
    assertNull(streamApp);
    config = new MapConfig(new HashMap<>());
    streamApp = TaskFactoryUtil.createStreamApplication(config);
    assertNull(streamApp);
}

Example 64 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestStreamProcessor method testStreamProcessorWithStreamTaskFactory.

/**
   * Should be able to create task instances from the provided task factory.
   */
@Test
public void testStreamProcessorWithStreamTaskFactory() {
    final String testSystem = "test-system";
    final String inputTopic = "numbers2";
    final String outputTopic = "output2";
    final int messageCount = 20;
    final Config configs = new MapConfig(createConfigs("1", testSystem, inputTopic, outputTopic, messageCount));
    createTopics(inputTopic, outputTopic);
    final StreamTaskFactory stf = IdentityStreamTask::new;
    final StreamProcessor processor = new StreamProcessor(configs, new HashMap<>(), stf, listener);
    produceMessages(inputTopic, messageCount);
    run(processor, endLatch);
    verifyNumMessages(outputTopic, messageCount);
}

Example 65 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class MockTaskProxy method getJobModel.

@Override
protected JobModel getJobModel(JobInstance jobInstance) {
    if (jobInstance.getJobId().contains("Bad") || jobInstance.getJobName().contains("Bad")) {
        throw new IllegalArgumentException("No tasks found.");
    }
    TaskModel task1Model = new TaskModel(new TaskName(TASK_1_NAME), SYSTEM_STREAM_PARTITIONS, CHANGE_LOG_PARTITION);
    TaskModel task2Model = new TaskModel(new TaskName(TASK_2_NAME), SYSTEM_STREAM_PARTITIONS, CHANGE_LOG_PARTITION);
    ContainerModel task1ContainerModel = new ContainerModel(TASK_1_CONTAINER_ID, 1, ImmutableMap.of(new TaskName(TASK_1_NAME), task1Model));
    ContainerModel task2ContainerModel = new ContainerModel(TASK_2_CONTAINER_ID, 2, ImmutableMap.of(new TaskName(TASK_2_NAME), task2Model));
    return new JobModel(new MapConfig(), ImmutableMap.of(TASK_1_CONTAINER_ID, task1ContainerModel, TASK_2_CONTAINER_ID, task2ContainerModel));
}