Examples with NodeId org.apache.hadoop.yarn.api.records.NodeId used on opensource projects

Example 61 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMWebServicesNodes method getNewRMNode.

private RMNodeImpl getNewRMNode(String host, int port, int memory) {
    NodeId nodeId = NodeId.newInstance(host, port);
    RMNodeImpl nodeImpl = new RMNodeImpl(nodeId, rm.getRMContext(), nodeId.getHost(), nodeId.getPort(), nodeId.getPort() + 1, RackResolver.resolve(nodeId.getHost()), Resource.newInstance(memory, 4), YarnVersionInfo.getVersion());
    rm.getRMContext().getRMNodes().put(nodeId, nodeImpl);
    return nodeImpl;
}

Example 62 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestRMWebServicesNodes method testNodesDefaultWithUnHealthyNode.

@Test
public void testNodesDefaultWithUnHealthyNode() throws JSONException, Exception {
    WebResource r = resource();
    getRunningRMNode("h1", 1234, 5120);
    // h2 will be in NEW state
    getNewRMNode("h2", 1235, 5121);
    RMNode node3 = getRunningRMNode("h3", 1236, 5122);
    NodeId nodeId3 = node3.getNodeID();
    RMNode node = rm.getRMContext().getRMNodes().get(nodeId3);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false, "test health report", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId3, 1, new ArrayList<ContainerStatus>(), null, nodeHealth, null, null, null);
    ((RMNodeImpl) node).handle(new RMNodeStatusEvent(nodeId3, nodeStatus, null));
    rm.waitForState(nodeId3, NodeState.UNHEALTHY);
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("nodes").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject nodes = json.getJSONObject("nodes");
    assertEquals("incorrect number of elements", 1, nodes.length());
    JSONArray nodeArray = nodes.getJSONArray("node");
    // 3 nodes, including the unhealthy node and the new node.
    assertEquals("incorrect number of elements", 3, nodeArray.length());
}

Example 63 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestConverterUtils method testNodeIdWithDefaultPort.

@Test
public void testNodeIdWithDefaultPort() throws URISyntaxException {
    NodeId nid;
    nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
    assertEquals(nid.getPort(), 10);
    assertEquals(nid.getHost(), "node");
    nid = ConverterUtils.toNodeIdWithDefaultPort("node");
    assertEquals(nid.getPort(), 0);
    assertEquals(nid.getHost(), "node");
}

Example 64 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestYARNTokenIdentifier method testNMTokenIdentifier.

@Test
public void testNMTokenIdentifier() throws IOException {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
    NodeId nodeId = NodeId.newInstance("host0", 0);
    String applicationSubmitter = "usr0";
    int masterKeyId = 1;
    NMTokenIdentifier token = new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter, masterKeyId);
    NMTokenIdentifier anotherToken = new NMTokenIdentifier();
    byte[] tokenContent = token.getBytes();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenContent, tokenContent.length);
    anotherToken.readFields(dib);
    // verify the whole record equals with original record
    Assert.assertEquals("Token is not the same after serialization " + "and deserialization.", token, anotherToken);
    // verify all properties are the same as original
    Assert.assertEquals("appAttemptId from proto is not the same with original token", anotherToken.getApplicationAttemptId(), appAttemptId);
    Assert.assertEquals("NodeId from proto is not the same with original token", anotherToken.getNodeId(), nodeId);
    Assert.assertEquals("applicationSubmitter from proto is not the same with original token", anotherToken.getApplicationSubmitter(), applicationSubmitter);
    Assert.assertEquals("masterKeyId from proto is not the same with original token", anotherToken.getKeyId(), masterKeyId);
}

Example 65 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class ContainerManagerImpl method serviceStart.

@Override
protected void serviceStart() throws Exception {
    // Enqueue user dirs in deletion context
    Configuration conf = getConfig();
    final InetSocketAddress initialAddress = conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS, YarnConfiguration.DEFAULT_NM_PORT);
    boolean usingEphemeralPort = (initialAddress.getPort() == 0);
    if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
        throw new IllegalArgumentException("Cannot support recovery with an " + "ephemeral server port. Check the setting of " + YarnConfiguration.NM_ADDRESS);
    }
    // If recovering then delay opening the RPC service until the recovery
    // of resources and containers have completed, otherwise requests from
    // clients during recovery can interfere with the recovery process.
    final boolean delayedRpcServerStart = context.getNMStateStore().canRecover();
    Configuration serverConf = new Configuration(conf);
    // always enforce it to be token-based.
    serverConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, SaslRpcServer.AuthMethod.TOKEN.toString());
    YarnRPC rpc = YarnRPC.create(conf);
    server = rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, serverConf, this.context.getNMTokenSecretManager(), conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
    // Enable service authorization?
    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
        refreshServiceAcls(conf, new NMPolicyProvider());
    }
    LOG.info("Blocking new container-requests as container manager rpc" + " server is still starting.");
    this.setBlockNewContainerRequests(true);
    String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
    String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
    String hostOverride = null;
    if (bindHost != null && !bindHost.isEmpty() && nmAddress != null && !nmAddress.isEmpty()) {
        //a bind-host case with an address, to support overriding the first
        //hostname found when querying for our hostname with the specified
        //address, combine the specified address with the actual port listened
        //on by the server
        hostOverride = nmAddress.split(":")[0];
    }
    // setup node ID
    InetSocketAddress connectAddress;
    if (delayedRpcServerStart) {
        connectAddress = NetUtils.getConnectAddress(initialAddress);
    } else {
        server.start();
        connectAddress = NetUtils.getConnectAddress(server);
    }
    NodeId nodeId = buildNodeId(connectAddress, hostOverride);
    ((NodeManager.NMContext) context).setNodeId(nodeId);
    this.context.getNMTokenSecretManager().setNodeId(nodeId);
    this.context.getContainerTokenSecretManager().setNodeId(nodeId);
    // start remaining services
    super.serviceStart();
    if (delayedRpcServerStart) {
        waitForRecoveredContainers();
        server.start();
        // check that the node ID is as previously advertised
        connectAddress = NetUtils.getConnectAddress(server);
        NodeId serverNode = buildNodeId(connectAddress, hostOverride);
        if (!serverNode.equals(nodeId)) {
            throw new IOException("Node mismatch after server started, expected '" + nodeId + "' but found '" + serverNode + "'");
        }
    }
    LOG.info("ContainerManager started at " + connectAddress);
    LOG.info("ContainerManager bound to " + initialAddress);
}