Examples with StartContainersRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest used on opensource projects

Example 26 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManagerSecurity method startContainer.

private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception {
    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(context, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    ContainerManagementProtocol proxy = null;
    try {
        proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
        StartContainersResponse response = proxy.startContainers(allRequests);
        for (SerializedException ex : response.getFailedRequests().values()) {
            parseAndThrowException(ex.deSerialize());
        }
    } finally {
        if (proxy != null) {
            rpc.stopProxy(proxy, conf);
        }
    }
}

Example 27 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class NMClientImpl method startContainer.

@Override
public Map<String, ByteBuffer> startContainer(Container container, ContainerLaunchContext containerLaunchContext) throws YarnException, IOException {
    // Do synchronization on StartedContainer to prevent race condition
    // between startContainer and stopContainer only when startContainer is
    // in progress for a given container.
    StartedContainer startingContainer = new StartedContainer(container.getId(), container.getNodeId());
    synchronized (startingContainer) {
        addStartingContainer(startingContainer);
        Map<String, ByteBuffer> allServiceResponse;
        ContainerManagementProtocolProxyData proxy = null;
        try {
            proxy = cmProxy.getProxy(container.getNodeId().toString(), container.getId());
            StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, container.getContainerToken());
            List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
            list.add(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
            StartContainersResponse response = proxy.getContainerManagementProtocol().startContainers(allRequests);
            if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(container.getId())) {
                Throwable t = response.getFailedRequests().get(container.getId()).deSerialize();
                parseAndThrowException(t);
            }
            allServiceResponse = response.getAllServicesMetaData();
            startingContainer.state = ContainerState.RUNNING;
        } catch (YarnException | IOException e) {
            startingContainer.state = ContainerState.COMPLETE;
            // Remove the started container if it failed to start
            startedContainers.remove(startingContainer.containerId);
            throw e;
        } catch (Throwable t) {
            startingContainer.state = ContainerState.COMPLETE;
            startedContainers.remove(startingContainer.containerId);
            throw RPCUtil.getRemoteException(t);
        } finally {
            if (proxy != null) {
                cmProxy.mayBeCloseProxy(proxy);
            }
        }
        return allServiceResponse;
    }
}

Example 28 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerLaunchRPC method testRPCTimeout.

private void testRPCTimeout(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, server.getListenerAddress(), conf);
        ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
        StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
        list.add(scRequest);
        StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
        try {
            proxy.startContainers(allRequests);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            Assert.assertEquals("Error, exception is not: " + SocketTimeoutException.class.getName(), SocketTimeoutException.class.getName(), e.getClass().getName());
            return;
        }
    } finally {
        server.stop();
    }
    Assert.fail("timeout exception should have occurred!");
}

Example 29 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestRPC method test.

private void test(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, NetUtils.getConnectAddress(server), conf);
    ContainerLaunchContext containerLaunchContext = RECORD_FACTORY.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
    Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    proxy.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = proxy.getContainerStatuses(gcsRequest);
    List<ContainerStatus> statuses = response.getContainerStatuses();
    //test remote exception
    boolean exception = false;
    try {
        StopContainersRequest stopRequest = RECORD_FACTORY.newRecordInstance(StopContainersRequest.class);
        stopRequest.setContainerIds(containerIds);
        proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
        exception = true;
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
        System.out.println("Test Exception is " + e.getMessage());
    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        server.stop();
    }
    Assert.assertTrue(exception);
    Assert.assertNotNull(statuses.get(0));
    Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}

Example 30 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManager method testLocalingResourceWhileContainerRunning.

// Start the container
// While the container is running, localize new resources.
// Verify the symlink is created properly
@Test
public void testLocalingResourceWhileContainerRunning() throws Exception {
    // Real del service
    delSrvc = new DeletionService(exec);
    delSrvc.init(conf);
    ((NodeManager.NMContext) context).setContainerExecutor(exec);
    containerManager = createContainerManager(delSrvc);
    containerManager.init(conf);
    containerManager.start();
    // set up local resources
    Map<String, LocalResource> localResource = setupLocalResources("file", "symLink1");
    ContainerLaunchContext context = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    context.setLocalResources(localResource);
    // a long running container - sleep
    context.setCommands(Arrays.asList("sleep 6"));
    ContainerId cId = createContainerId(0);
    // start the container
    StartContainerRequest scRequest = StartContainerRequest.newInstance(context, createContainerToken(cId, DUMMY_RM_IDENTIFIER, this.context.getNodeId(), user, this.context.getContainerTokenSecretManager()));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(Arrays.asList(scRequest));
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.RUNNING);
    BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getApplicationAttemptId().getApplicationId(), ApplicationState.RUNNING);
    checkResourceLocalized(cId, "symLink1");
    // Localize new local resources while container is running
    Map<String, LocalResource> localResource2 = setupLocalResources("file2", "symLink2");
    ResourceLocalizationRequest request = ResourceLocalizationRequest.newInstance(cId, localResource2);
    containerManager.localize(request);
    // Verify resource is localized and symlink is created.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            try {
                checkResourceLocalized(cId, "symLink2");
                return true;
            } catch (Throwable e) {
                return false;
            }
        }
    }, 500, 20000);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
    // Verify container cannot localize resources while at non-running state.
    try {
        containerManager.localize(request);
        Assert.fail();
    } catch (YarnException e) {
        Assert.assertTrue(e.getMessage().contains("Cannot perform LOCALIZE"));
    }
}