Examples with ResourceIsolationInterface org.apache.storm.container.ResourceIsolationInterface used on opensource projects

Example 1 with ResourceIsolationInterface

use of org.apache.storm.container.ResourceIsolationInterface in project storm by apache.

the class ContainerLauncher method make.

/**
     * Factory to create the right container launcher 
     * for the config and the environment.
     * @param conf the config
     * @param supervisorId the ID of the supervisor
     * @param sharedContext Used in local mode to let workers talk together without netty
     * @return the proper container launcher
     * @throws IOException on any error
     */
public static ContainerLauncher make(Map<String, Object> conf, String supervisorId, IContext sharedContext) throws IOException {
    if (ConfigUtils.isLocalMode(conf)) {
        return new LocalContainerLauncher(conf, supervisorId, sharedContext);
    }
    ResourceIsolationInterface resourceIsolationManager = null;
    if (Utils.getBoolean(conf.get(Config.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE), false)) {
        resourceIsolationManager = Utils.newInstance((String) conf.get(Config.STORM_RESOURCE_ISOLATION_PLUGIN));
        resourceIsolationManager.prepare(conf);
        LOG.info("Using resource isolation plugin {} {}", conf.get(Config.STORM_RESOURCE_ISOLATION_PLUGIN), resourceIsolationManager);
    }
    if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
        return new RunAsUserContainerLauncher(conf, supervisorId, resourceIsolationManager);
    }
    return new BasicContainerLauncher(conf, supervisorId, resourceIsolationManager);
}

Example 2 with ResourceIsolationInterface

use of org.apache.storm.container.ResourceIsolationInterface in project storm by apache.

the class ContainerLauncher method make.

/**
 * Factory to create the right container launcher
 * for the config and the environment.
 * @param conf the config
 * @param supervisorId the ID of the supervisor
 * @param supervisorPort the parent supervisor thrift server port
 * @param sharedContext Used in local mode to let workers talk together without netty
 * @param metricsRegistry The metrics registry.
 * @param containerMemoryTracker The shared memory tracker for the supervisor's containers
 * @param localSupervisor The local supervisor Thrift interface. Only used for local clusters, distributed clusters use Thrift directly.
 * @return the proper container launcher
 * @throws IOException on any error
 */
public static ContainerLauncher make(Map<String, Object> conf, String supervisorId, int supervisorPort, IContext sharedContext, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker, org.apache.storm.generated.Supervisor.Iface localSupervisor) throws IOException {
    if (ConfigUtils.isLocalMode(conf)) {
        return new LocalContainerLauncher(conf, supervisorId, supervisorPort, sharedContext, metricsRegistry, containerMemoryTracker, localSupervisor);
    }
    ResourceIsolationInterface resourceIsolationManager;
    if (ObjectReader.getBoolean(conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE), false)) {
        resourceIsolationManager = ReflectionUtils.newInstance((String) conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN));
        LOG.info("Using resource isolation plugin {}: {}", conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN), resourceIsolationManager);
    } else {
        resourceIsolationManager = new DefaultResourceIsolationManager();
        LOG.info("{} is false. Using default resource isolation plugin: {}", DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE, resourceIsolationManager);
    }
    resourceIsolationManager.prepare(conf);
    return new BasicContainerLauncher(conf, supervisorId, supervisorPort, resourceIsolationManager, metricsRegistry, containerMemoryTracker);
}

Example 3 with ResourceIsolationInterface

use of org.apache.storm.container.ResourceIsolationInterface in project storm by apache.

the class ContainerTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    final int supervisorPort = 6628;
    final int port = 8080;
    final String topoId = "test_topology";
    final String workerId = "worker_id";
    final String user = "me";
    final String stormLocal = asAbsPath("tmp", "testing");
    final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
    final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
    final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
    final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
    final Map<String, Object> topoConf = new HashMap<>();
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    final StringWriter yamlDump = new StringWriter();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.fileExists(workerArtifacts)).thenReturn(true);
    when(ops.fileExists(workerRoot)).thenReturn(true);
    when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
    ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
    when(iso.isResourceManaged()).thenReturn(true);
    LocalAssignment la = new LocalAssignment();
    la.set_owner(user);
    la.set_topology_id(topoId);
    MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, workerId, topoConf, ops, new StormMetricsRegistry());
    mc.cleanUp();
    verify(iso).cleanup(user, workerId, port);
    verify(ops).deleteIfExists(eq(new File(workerRoot, "pids")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(new File(workerRoot, "tmp")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(new File(workerRoot, "heartbeats")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(workerRoot), eq(user), any(String.class));
    verify(ops).deleteIfExists(workerUserFile);
}

Example 4 with ResourceIsolationInterface

use of org.apache.storm.container.ResourceIsolationInterface in project storm by apache.

the class ContainerTest method testSetup.

@SuppressWarnings("unchecked")
@Test
public void testSetup() throws Exception {
    final int port = 8080;
    final String topoId = "test_topology";
    final String workerId = "worker_id";
    final String user = "me";
    final String stormLocal = asAbsPath("tmp", "testing");
    final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
    final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
    final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
    final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
    final File distRoot = asAbsFile(stormLocal, "supervisor", "stormdist", topoId);
    final Map<String, Object> topoConf = new HashMap<>();
    final List<String> topoUsers = Arrays.asList("t-user-a", "t-user-b");
    final List<String> logUsers = Arrays.asList("l-user-a", "l-user-b");
    final List<String> topoGroups = Arrays.asList("t-group-a", "t-group-b");
    final List<String> logGroups = Arrays.asList("l-group-a", "l-group-b");
    topoConf.put(DaemonConfig.LOGS_GROUPS, logGroups);
    topoConf.put(Config.TOPOLOGY_GROUPS, topoGroups);
    topoConf.put(DaemonConfig.LOGS_USERS, logUsers);
    topoConf.put(Config.TOPOLOGY_USERS, topoUsers);
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    final StringWriter yamlDump = new StringWriter();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.fileExists(workerArtifacts)).thenReturn(true);
    when(ops.fileExists(workerRoot)).thenReturn(true);
    when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    la.set_owner(user);
    ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
    MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", 6628, 8080, la, iso, workerId, topoConf, ops, new StormMetricsRegistry());
    mc.setup();
    // Initial Setup
    verify(ops).forceMkdir(new File(workerRoot, "pids"));
    verify(ops).forceMkdir(new File(workerRoot, "tmp"));
    verify(ops).forceMkdir(new File(workerRoot, "heartbeats"));
    verify(ops).fileExists(workerArtifacts);
    // Log file permissions
    verify(ops).getWriter(logMetadataFile);
    String yamlResult = yamlDump.toString();
    Yaml yaml = new Yaml();
    Map<String, Object> result = yaml.load(yamlResult);
    assertEquals(workerId, result.get("worker-id"));
    assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER));
    HashSet<String> allowedUsers = new HashSet<>(topoUsers);
    allowedUsers.addAll(logUsers);
    assertEquals(allowedUsers, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_USERS))));
    HashSet<String> allowedGroups = new HashSet<>(topoGroups);
    allowedGroups.addAll(logGroups);
    assertEquals(allowedGroups, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_GROUPS))));
    // Save the current user to help with recovery
    verify(ops).dump(workerUserFile, user);
    // Create links to artifacts dir
    verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
    // Create links to blobs
    verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
}

Example 5 with ResourceIsolationInterface

use of org.apache.storm.container.ResourceIsolationInterface in project storm by apache.

the class BasicContainerTest method testCleanUp.

@Test
public void testCleanUp() throws Exception {
    final String topoId = "test_topology";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String workerId = "worker-id";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    Map<String, Object> superConf = new HashMap<>();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    Map<String, Integer> workerState = new HashMap<String, Integer>();
    workerState.put(workerId, port);
    LocalState ls = mock(LocalState.class);
    when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
    ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
    MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
    mc.cleanUp();
    assertNull(mc.workerId);
    verify(ls).getApprovedWorkers();
    Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
    verify(ls).setApprovedWorkers(expectedNewState);
}