Examples with StopContainersRequest org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest used on opensource projects

Example 6 with StopContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest in project hadoop by apache.

the class Application method finishTask.

public synchronized void finishTask(Task task) throws IOException, YarnException {
    Set<Task> tasks = this.tasks.get(task.getSchedulerKey());
    if (!tasks.remove(task)) {
        throw new IllegalStateException("Finishing unknown task " + task.getTaskId() + " from application " + applicationId);
    }
    NodeManager nodeManager = task.getNodeManager();
    ContainerId containerId = task.getContainerId();
    task.stop();
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    nodeManager.stopContainers(stopRequest);
    Resources.subtractFrom(used, requestSpec.get(task.getSchedulerKey()));
    LOG.info("Finished task " + task.getTaskId() + " of application " + applicationId + " on node " + nodeManager.getHostName() + ", currently using " + used + " resources");
}

Example 7 with StopContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest in project hadoop by apache.

the class TestContainerManagerSecurity method stopContainer.

private void stopContainer(YarnRPC rpc, Token nmToken, List<ContainerId> containerId, ApplicationAttemptId appAttemptId, NodeId nodeId) throws Exception {
    StopContainersRequest request = StopContainersRequest.newInstance(containerId);
    ContainerManagementProtocol proxy = null;
    try {
        proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString());
        StopContainersResponse response = proxy.stopContainers(request);
        if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(containerId)) {
            parseAndThrowException(response.getFailedRequests().get(containerId).deSerialize());
        }
    } catch (Exception e) {
        if (proxy != null) {
            rpc.stopProxy(proxy, conf);
        }
    }
}

Example 8 with StopContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest in project hadoop by apache.

the class TestRPC method test.

private void test(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, NetUtils.getConnectAddress(server), conf);
    ContainerLaunchContext containerLaunchContext = RECORD_FACTORY.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
    Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    proxy.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = proxy.getContainerStatuses(gcsRequest);
    List<ContainerStatus> statuses = response.getContainerStatuses();
    //test remote exception
    boolean exception = false;
    try {
        StopContainersRequest stopRequest = RECORD_FACTORY.newRecordInstance(StopContainersRequest.class);
        stopRequest.setContainerIds(containerIds);
        proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
        exception = true;
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
        System.out.println("Test Exception is " + e.getMessage());
    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        server.stop();
    }
    Assert.assertTrue(exception);
    Assert.assertNotNull(statuses.get(0));
    Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}

Example 9 with StopContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest in project hadoop by apache.

the class TestContainerManager method testUnauthorizedRequests.

@Test
public void testUnauthorizedRequests() throws IOException, YarnException {
    containerManager.start();
    // Create a containerId that belongs to an unauthorized appId
    ContainerId cId = createContainerId(0, 1);
    // startContainers()
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse startResponse = containerManager.startContainers(allRequests);
    Assert.assertFalse("Should not be authorized to start container", startResponse.getSuccessfullyStartedContainers().contains(cId));
    Assert.assertTrue("Start container request should fail", startResponse.getFailedRequests().containsKey(cId));
    // Insert the containerId into context, make it as if it is running
    ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(scRequest.getContainerToken());
    Container container = new ContainerImpl(conf, null, containerLaunchContext, null, metrics, containerTokenIdentifier, context);
    context.getContainers().put(cId, container);
    // stopContainers()
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertFalse("Should not be authorized to stop container", stopResponse.getSuccessfullyStoppedContainers().contains(cId));
    Assert.assertTrue("Stop container request should fail", stopResponse.getFailedRequests().containsKey(cId));
    // getContainerStatuses()
    containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = containerManager.getContainerStatuses(request);
    Assert.assertEquals("Should not be authorized to get container status", response.getContainerStatuses().size(), 0);
    Assert.assertTrue("Get status request should fail", response.getFailedRequests().containsKey(cId));
}

Example 10 with StopContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest in project hadoop by apache.

the class TestContainerLaunch method internalKillTest.

private void internalKillTest(boolean delayed) throws Exception {
    conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, delayed ? 1000 : 0);
    containerManager.start();
    // ////// Construct the Container-id
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
    File processStartFile = new File(tmpDir, "pid.txt").getAbsoluteFile();
    // setup a script that can handle sigterm gracefully
    File scriptFile = Shell.appendScriptExtension(tmpDir, "testscript");
    PrintWriter writer = new PrintWriter(new FileOutputStream(scriptFile));
    if (Shell.WINDOWS) {
        writer.println("@echo \"Running testscript for delayed kill\"");
        writer.println("@echo \"Writing pid to start file\"");
        writer.println("@echo " + cId + "> " + processStartFile);
        writer.println("@ping -n 100 127.0.0.1 >nul");
    } else {
        writer.println("#!/bin/bash\n\n");
        writer.println("echo \"Running testscript for delayed kill\"");
        writer.println("hello=\"Got SIGTERM\"");
        writer.println("umask 0");
        writer.println("trap \"echo $hello >> " + processStartFile + "\" SIGTERM");
        writer.println("echo \"Writing pid to start file\"");
        writer.println("echo $$ >> " + processStartFile);
        writer.println("while true; do\nsleep 1s;\ndone");
    }
    writer.close();
    FileUtil.setExecutable(scriptFile, true);
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    // upload the script file so that the container can run it
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file.sh";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    // set up the rest of the container
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    Priority priority = Priority.newInstance(10);
    long createTime = 1234;
    Token containerToken = createContainerToken(cId, priority, createTime);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    int timeoutSecs = 0;
    while (!processStartFile.exists() && timeoutSecs++ < 20) {
        Thread.sleep(1000);
        LOG.info("Waiting for process start-file to be created");
    }
    Assert.assertTrue("ProcessStartFile doesn't exist!", processStartFile.exists());
    NMContainerStatus nmContainerStatus = containerManager.getContext().getContainers().get(cId).getNMContainerStatus();
    Assert.assertEquals(priority, nmContainerStatus.getPriority());
    // Now test the stop functionality.
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    containerManager.stopContainers(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
    // if delayed container stop sends a sigterm followed by a sigkill
    // otherwise sigkill is sent immediately 
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    Assert.assertEquals(ContainerExitStatus.KILLED_BY_APPMASTER, containerStatus.getExitStatus());
    // verify that the job object with ID matching container ID no longer exists.
    if (Shell.WINDOWS || !delayed) {
        Assert.assertFalse("Process is still alive!", DefaultContainerExecutor.containerIsAlive(cId.toString()));
    } else {
        BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
        boolean foundSigTermMessage = false;
        while (true) {
            String line = reader.readLine();
            if (line == null) {
                break;
            }
            if (line.contains("SIGTERM")) {
                foundSigTermMessage = true;
                break;
            }
        }
        Assert.assertTrue("Did not find sigterm message", foundSigTermMessage);
        reader.close();
    }
}