Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 71 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalizerResourceMapper method testResourceMapWithInvalidTypeFailure.

@Test
public void testResourceMapWithInvalidTypeFailure() {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("No enum constant org.apache.hadoop.yarn.api.records.LocalResourceType.INVALIDTYPE");
    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "invalidType");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");
    Config conf = new MapConfig(configMap);
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
}

Example 72 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestYarnJobFactory method testGetJobWithFsImplOverride.

@Test
public void testGetJobWithFsImplOverride() {
    YarnJobFactory jobFactory = new YarnJobFactory();
    YarnJob yarnJob = jobFactory.getJob(new MapConfig(ImmutableMap.of("fs.http.impl", "org.apache.myHttp", "fs.myscheme.impl", "org.apache.myScheme")));
    Configuration hConfig = yarnJob.client().yarnClient().getConfig();
    assertEquals("org.apache.myHttp", hConfig.get("fs.http.impl"));
    assertEquals("org.apache.myScheme", hConfig.get("fs.myscheme.impl"));
}

Example 73 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalStoreMonitor method shouldContinueLocalStoreCleanUpAfterFailureToCleanUpStoreOfAJob.

@Test
public void shouldContinueLocalStoreCleanUpAfterFailureToCleanUpStoreOfAJob() throws Exception {
    File testFailingJobDir = new File(System.getProperty("java.io.tmpdir") + File.separator + "samza-test-job/", "test-jobName-jobId-1");
    File testFailingTaskStoreDir = new File(new File(testFailingJobDir, "test-store"), "test-task");
    FileUtils.forceMkdir(testFailingTaskStoreDir);
    // For job: test-jobName-jobId-1, throw up in getTasks call and
    // expect the cleanup to succeed for other job: test-jobName-jobId.
    Mockito.doThrow(new RuntimeException("Dummy exception message.")).when(jobsClientMock).getTasks(new JobInstance("test-jobName", "jobId-1"));
    Task task = new Task("notLocalHost", "test-task", "0", new ArrayList<>(), ImmutableList.of("test-store"));
    Mockito.when(jobsClientMock.getTasks(new JobInstance("test-jobName", "jobId"))).thenReturn(ImmutableList.of(task));
    Map<String, String> configMap = new HashMap<>(config);
    configMap.put(LocalStoreMonitorConfig.CONFIG_IGNORE_FAILURES, "true");
    LocalStoreMonitor localStoreMonitor = new LocalStoreMonitor(new LocalStoreMonitorConfig(new MapConfig(configMap)), localStoreMonitorMetrics, jobsClientMock);
    localStoreMonitor.monitor();
    // Non failing job directory should be cleaned up.
    assertTrue("Task store directory should not exist.", !taskStoreDir.exists());
    FileUtils.deleteDirectory(testFailingJobDir);
}

Example 74 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalStoreMonitor method setUp.

@Before
public void setUp() throws Exception {
    // Make scaffold directories for testing.
    FileUtils.forceMkdir(taskStoreDir);
    taskStoreSize = taskStoreDir.getTotalSpace();
    // Set default return values for methods.
    Mockito.when(jobsClientMock.getJobStatus(Mockito.any())).thenReturn(JobStatus.STOPPED);
    Task task = new Task("localHost", "test-task", "0", new ArrayList<>(), ImmutableList.of("test-store"));
    Mockito.when(jobsClientMock.getTasks(Mockito.any())).thenReturn(ImmutableList.of(task));
    localStoreMonitorMetrics = new LocalStoreMonitorMetrics("TestMonitorName", new NoOpMetricsRegistry());
    // Initialize the local store monitor with mock and config
    localStoreMonitor = new LocalStoreMonitor(new LocalStoreMonitorConfig(new MapConfig(config)), localStoreMonitorMetrics, jobsClientMock);
}

Example 75 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestRocksDbKeyValueReader method testReadCorrectDbValue.

@Test
public void testReadCorrectDbValue() throws RocksDBException {
    HashMap<String, String> map = new HashMap<String, String>();
    map.put("stores." + DB_NAME + ".factory", "mockFactory");
    map.put("stores." + DB_NAME + ".key.serde", "string");
    map.put("stores." + DB_NAME + ".msg.serde", "string");
    Config config = new MapConfig(map);
    RocksDbKeyValueReader reader = new RocksDbKeyValueReader(DB_NAME, dirPath.toString(), config);
    assertEquals("this is string", reader.get("testString"));
    // should throw exception if the input is in other type
    boolean throwClassCastException = false;
    try {
        reader.get(123);
    } catch (Exception e) {
        if (e instanceof ClassCastException) {
            throwClassCastException = true;
        }
    }
    assertTrue(throwClassCastException);
    reader.stop();
    // test with customized serde
    map.put("serializers.registry.mock.class", IntegerSerdeFactory.class.getCanonicalName());
    map.put("stores." + DB_NAME + ".key.serde", "mock");
    map.put("stores." + DB_NAME + ".msg.serde", "mock");
    config = new MapConfig(map);
    reader = new RocksDbKeyValueReader(DB_NAME, dirPath.toString(), config);
    assertEquals(456, reader.get(123));
    assertNull(reader.get(789));
    reader.stop();
}