Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 46 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestYarnJobFactory method testGetJobWithDefaultFsImpl.

@Test
public void testGetJobWithDefaultFsImpl() {
    YarnJobFactory jobFactory = new YarnJobFactory();
    YarnJob yarnJob = jobFactory.getJob(new MapConfig());
    Configuration hConfig = yarnJob.client().yarnClient().getConfig();
    assertEquals(HttpFileSystem.class.getName(), hConfig.get("fs.http.impl"));
    assertEquals(HttpFileSystem.class.getName(), hConfig.get("fs.https.impl"));
}

Example 47 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestLocalizerResourceMapper method testResourceMapSuccess.

@Test
public void testResourceMapSuccess() {
    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");
    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");
    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");
    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");
    Config conf = new MapConfig(configMap);
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();
    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());
    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());
    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());
    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());
    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either", resourceMap.get("notExisting"));
}

Example 48 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestYarnJobFactory method testGetJobWithFsImplSubkeys.

@Test
public void testGetJobWithFsImplSubkeys() {
    YarnJobFactory jobFactory = new YarnJobFactory();
    YarnJob yarnJob = jobFactory.getJob(new MapConfig(ImmutableMap.of("fs.myscheme.impl", "org.apache.myScheme", "fs.myscheme.impl.client", "org.apache.mySchemeClient")));
    Configuration hConfig = yarnJob.client().yarnClient().getConfig();
    assertEquals("org.apache.myScheme", hConfig.get("fs.myscheme.impl"));
    assertEquals("org.apache.mySchemeClient", hConfig.get("fs.myscheme.impl.client"));
}

Example 49 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestCoordinatorStreamWriter method testCoordinatorStream.

@Test
public void testCoordinatorStream() {
    Map<String, String> configMap = new HashMap<>();
    configMap.put("systems.coordinatorStreamWriter.samza.factory", "org.apache.samza.coordinator.stream.MockCoordinatorStreamSystemFactory");
    configMap.put("job.name", "coordinator-stream-writer-test");
    configMap.put("job.coordinator.system", "coordinatorStreamWriter");
    Config config = new MapConfig(configMap);
    coordinatorStreamWriter = new CoordinatorStreamWriter(config);
    boolean exceptionHappened = false;
    try {
        //get coordinator system producer
        Field coordinatorProducerField = coordinatorStreamWriter.getClass().getDeclaredField("coordinatorStreamSystemProducer");
        coordinatorProducerField.setAccessible(true);
        assertNotNull(coordinatorProducerField.get(coordinatorStreamWriter));
        CoordinatorStreamSystemProducer coordinatorStreamSystemProducer = (CoordinatorStreamSystemProducer) coordinatorProducerField.get(coordinatorStreamWriter);
        //get mock system producer
        Field systemProducerField = coordinatorStreamSystemProducer.getClass().getDeclaredField("systemProducer");
        systemProducerField.setAccessible(true);
        systemProducer = (MockCoordinatorStreamSystemFactory.MockSystemProducer) systemProducerField.get(coordinatorStreamSystemProducer);
        testStart();
        testSendMessage();
        testStop();
    } catch (NoSuchFieldException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
        e.printStackTrace();
        exceptionHappened = true;
    }
    assertFalse(exceptionHappened);
}

Example 50 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestContainerProcessManager method testNewContainerRequestedOnFailureWithKnownCode.

/**
   * Test AM requests a new container when a task fails
   * Error codes with same behavior - Disk failure, preemption and aborted
   */
@Test
public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
    Config conf = getConfig();
    Map<String, String> config = new HashMap<>();
    config.putAll(getConfig());
    config.remove("yarn.container.retry.count");
    ContainerProcessManager taskManager = new ContainerProcessManager(new MapConfig(conf), state, new MetricsRegistryMap(), manager);
    MockContainerAllocator allocator = new MockContainerAllocator(manager, conf, state);
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);
    Thread thread = new Thread(allocator);
    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);
    // Start the task manager
    taskManager.start();
    assertFalse(taskManager.shouldShutdown());
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    SamzaResource container = new SamzaResource(1, 1000, "abc", "id1");
    taskManager.onResourceAllocated(container);
    // Allow container to run and update state
    Thread.sleep(300);
    // Create container failure - with ContainerExitStatus.DISKS_FAILED
    taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "Disk failure", SamzaResourceStatus.DISK_FAIL));
    // The above failure should trigger a container request
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(2, manager.resourceRequests.size());
    assertEquals(0, manager.releasedResources.size());
    assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
    // Create container failure - with ContainerExitStatus.PREEMPTED
    taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "Preemption", SamzaResourceStatus.PREEMPTED));
    // The above failure should trigger a container request
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
    // Create container failure - with ContainerExitStatus.ABORTED
    taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "Aborted", SamzaResourceStatus.ABORTED));
    // The above failure should trigger a container request
    assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
    assertEquals(2, manager.resourceRequests.size());
    assertEquals(0, manager.releasedResources.size());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
    taskManager.stop();
}