Examples with StartContainerRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest used on opensource projects

Example 46 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testStopQueuedContainer.

/**
   * Start running one GUARANTEED container and queue two OPPORTUNISTIC ones.
   * Try killing one of the two queued containers.
   * @throws Exception
   */
@Test
public void testStopQueuedContainer() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    Thread.sleep(2000);
    // Assert there is initially one container running and two queued.
    int runningContainersNo = 0;
    int queuedContainersNo = 0;
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getState() == org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) {
            runningContainersNo++;
        } else if (status.getState() == org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
            queuedContainersNo++;
        }
        System.out.println("\nStatus : [" + status + "]\n");
    }
    Assert.assertEquals(1, runningContainersNo);
    Assert.assertEquals(2, queuedContainersNo);
    // Stop one of the two queued containers.
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(Arrays.asList(createContainerId(1)));
    containerManager.stopContainers(stopRequest);
    Thread.sleep(2000);
    // Assert queued container got properly stopped.
    statList.clear();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    statRequest = GetContainerStatusesRequest.newInstance(statList);
    HashMap<org.apache.hadoop.yarn.api.records.ContainerState, ContainerStatus> map = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
        for (ContainerStatus status : containerStatuses) {
            System.out.println("\nStatus : [" + status + "]\n");
            map.put(status.getState(), status);
            if (map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) && map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) && map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)) {
                break;
            }
            Thread.sleep(1000);
        }
    }
    Assert.assertEquals(createContainerId(0), map.get(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING).getContainerId());
    Assert.assertEquals(createContainerId(1), map.get(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE).getContainerId());
    Assert.assertEquals(createContainerId(2), map.get(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED).getContainerId());
}

Example 47 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testUnexpectedKeyDoesntThrowException.

@Test
public void testUnexpectedKeyDoesntThrowException() throws IOException {
    // test empty when no state
    List<RecoveredContainerState> recoveredContainers = stateStore.loadContainersState();
    assertTrue(recoveredContainers.isEmpty());
    // create a container request
    ApplicationId appId = ApplicationId.newInstance(1234, 3);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 4);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
    LocalResource lrsrc = LocalResource.newInstance(URL.newInstance("hdfs", "somehost", 12345, "/some/path/to/rsrc"), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 123L, 1234567890L);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put("rsrc", lrsrc);
    Map<String, String> env = new HashMap<String, String>();
    env.put("somevar", "someval");
    List<String> containerCmds = new ArrayList<String>();
    containerCmds.add("somecmd");
    containerCmds.add("somearg");
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    serviceData.put("someservice", ByteBuffer.wrap(new byte[] { 0x1, 0x2, 0x3 }));
    ByteBuffer containerTokens = ByteBuffer.wrap(new byte[] { 0x7, 0x8, 0x9, 0xa });
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>();
    acls.put(ApplicationAccessType.VIEW_APP, "viewuser");
    acls.put(ApplicationAccessType.MODIFY_APP, "moduser");
    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(localResources, env, containerCmds, serviceData, containerTokens, acls);
    Resource containerRsrc = Resource.newInstance(1357, 3);
    ContainerTokenIdentifier containerTokenId = new ContainerTokenIdentifier(containerId, "host", "user", containerRsrc, 9876543210L, 42, 2468, Priority.newInstance(7), 13579);
    Token containerToken = Token.newInstance(containerTokenId.getBytes(), ContainerTokenIdentifier.KIND.toString(), "password".getBytes(), "tokenservice");
    StartContainerRequest containerReq = StartContainerRequest.newInstance(clc, containerToken);
    stateStore.storeContainer(containerId, 0, containerReq);
    // add a invalid key
    byte[] invalidKey = ("ContainerManager/containers/" + containerId.toString() + "/invalidKey1234").getBytes();
    stateStore.getDB().put(invalidKey, new byte[1]);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    RecoveredContainerState rcs = recoveredContainers.get(0);
    assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertEquals(false, rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertTrue(rcs.getDiagnostics().isEmpty());
    assertEquals(RecoveredContainerType.KILL, rcs.getRecoveryType());
    // assert unknown keys are cleaned up finally
    assertNotNull(stateStore.getDB().get(invalidKey));
    stateStore.removeContainer(containerId);
    assertNull(stateStore.getDB().get(invalidKey));
}

Example 48 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class NMClientImpl method startContainer.

@Override
public Map<String, ByteBuffer> startContainer(Container container, ContainerLaunchContext containerLaunchContext) throws YarnException, IOException {
    // Do synchronization on StartedContainer to prevent race condition
    // between startContainer and stopContainer only when startContainer is
    // in progress for a given container.
    StartedContainer startingContainer = new StartedContainer(container.getId(), container.getNodeId());
    synchronized (startingContainer) {
        addStartingContainer(startingContainer);
        Map<String, ByteBuffer> allServiceResponse;
        ContainerManagementProtocolProxyData proxy = null;
        try {
            proxy = cmProxy.getProxy(container.getNodeId().toString(), container.getId());
            StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, container.getContainerToken());
            List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
            list.add(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
            StartContainersResponse response = proxy.getContainerManagementProtocol().startContainers(allRequests);
            if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(container.getId())) {
                Throwable t = response.getFailedRequests().get(container.getId()).deSerialize();
                parseAndThrowException(t);
            }
            allServiceResponse = response.getAllServicesMetaData();
            startingContainer.state = ContainerState.RUNNING;
        } catch (YarnException | IOException e) {
            startingContainer.state = ContainerState.COMPLETE;
            // Remove the started container if it failed to start
            startedContainers.remove(startingContainer.containerId);
            throw e;
        } catch (Throwable t) {
            startingContainer.state = ContainerState.COMPLETE;
            startedContainers.remove(startingContainer.containerId);
            throw RPCUtil.getRemoteException(t);
        } finally {
            if (proxy != null) {
                cmProxy.mayBeCloseProxy(proxy);
            }
        }
        return allServiceResponse;
    }
}

Example 49 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class StartContainersRequestPBImpl method initLocalRequests.

private void initLocalRequests() {
    StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder;
    List<StartContainerRequestProto> requestList = p.getStartContainerRequestList();
    this.requests = new ArrayList<StartContainerRequest>();
    for (StartContainerRequestProto r : requestList) {
        this.requests.add(convertFromProtoFormat(r));
    }
}

Example 50 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerLaunchRPC method testRPCTimeout.

private void testRPCTimeout(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, server.getListenerAddress(), conf);
        ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
        StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
        list.add(scRequest);
        StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
        try {
            proxy.startContainers(allRequests);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            Assert.assertEquals("Error, exception is not: " + SocketTimeoutException.class.getName(), SocketTimeoutException.class.getName(), e.getClass().getName());
            return;
        }
    } finally {
        server.stop();
    }
    Assert.fail("timeout exception should have occurred!");
}