Examples with NodeState org.apache.hadoop.yarn.api.records.NodeState used on opensource projects

Example 1 with NodeState

use of org.apache.hadoop.yarn.api.records.NodeState in project hadoop by apache.

the class NodeCLI method listDetailedClusterNodes.

/**
   * Lists the nodes which are matching the given node states along with
   * detailed node informations such as resource usage etc.
   *
   * @param nodeStates
   * @throws YarnException
   * @throws IOException
   */
private void listDetailedClusterNodes(Set<NodeState> nodeStates) throws YarnException, IOException {
    PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
    List<NodeReport> nodesReport = client.getNodeReports(nodeStates.toArray(new NodeState[0]));
    writer.println("Total Nodes:" + nodesReport.size());
    writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address", "Number-of-Running-Containers");
    for (NodeReport nodeReport : nodesReport) {
        writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport.getNodeState(), nodeReport.getHttpAddress(), nodeReport.getNumContainers());
        writer.println("Detailed Node Information :");
        writer.print("\tConfigured Resources : ");
        writer.println(nodeReport.getCapability());
        writer.print("\tAllocated Resources : ");
        if (nodeReport.getUsed() != null) {
            writer.print(nodeReport.getUsed());
        }
        writer.println();
        writer.print("\tResource Utilization by Node : ");
        if (nodeReport.getNodeUtilization() != null) {
            writer.print("PMem:" + nodeReport.getNodeUtilization().getPhysicalMemory() + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory() + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU());
        }
        writer.println();
        writer.print("\tResource Utilization by Containers : ");
        if (nodeReport.getAggregatedContainersUtilization() != null) {
            writer.print("PMem:" + nodeReport.getAggregatedContainersUtilization().getPhysicalMemory() + " MB, VMem:" + nodeReport.getAggregatedContainersUtilization().getVirtualMemory() + " MB, VCores:" + nodeReport.getAggregatedContainersUtilization().getCPU());
        }
        writer.println();
        writer.print("\tNode-Labels : ");
        // Create a List for node labels since we need it get sorted
        List<String> nodeLabelsList = new ArrayList<String>(nodeReport.getNodeLabels());
        Collections.sort(nodeLabelsList);
        writer.println(StringUtils.join(nodeLabelsList.iterator(), ','));
    }
    writer.flush();
}

Example 2 with NodeState

use of org.apache.hadoop.yarn.api.records.NodeState in project hadoop by apache.

the class NodeCLI method run.

@Override
public int run(String[] args) throws Exception {
    Options opts = new Options();
    opts.addOption(HELP_CMD, false, "Displays help for all commands.");
    opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
    opts.addOption(LIST_CMD, false, "List all running nodes. " + "Supports optional use of -states to filter nodes " + "based on node state, all -all to list all nodes, " + "-showDetails to display more details about each node.");
    Option nodeStateOpt = new Option(NODE_STATE_CMD, true, "Works with -list to filter nodes based on input comma-separated " + "list of node states. " + getAllValidNodeStates());
    nodeStateOpt.setValueSeparator(',');
    nodeStateOpt.setArgs(Option.UNLIMITED_VALUES);
    nodeStateOpt.setArgName("States");
    opts.addOption(nodeStateOpt);
    Option allOpt = new Option(NODE_ALL, false, "Works with -list to list all nodes.");
    opts.addOption(allOpt);
    Option showDetailsOpt = new Option(NODE_SHOW_DETAILS, false, "Works with -list to show more details about each node.");
    opts.addOption(showDetailsOpt);
    opts.getOption(STATUS_CMD).setArgName("NodeId");
    if (args != null && args.length > 0) {
        for (int i = args.length - 1; i >= 0; i--) {
            if (args[i].equalsIgnoreCase("-" + NODE_ALL)) {
                args[i] = "-" + NODE_ALL;
            }
        }
    }
    int exitCode = -1;
    CommandLine cliParser = null;
    try {
        cliParser = new GnuParser().parse(opts, args);
    } catch (MissingArgumentException ex) {
        sysout.println("Missing argument for options");
        printUsage(opts);
        return exitCode;
    }
    if (cliParser.hasOption("status")) {
        if (args.length != 2) {
            printUsage(opts);
            return exitCode;
        }
        printNodeStatus(cliParser.getOptionValue("status"));
    } else if (cliParser.hasOption("list")) {
        Set<NodeState> nodeStates = new HashSet<NodeState>();
        if (cliParser.hasOption(NODE_ALL)) {
            for (NodeState state : NodeState.values()) {
                nodeStates.add(state);
            }
        } else if (cliParser.hasOption(NODE_STATE_CMD)) {
            String[] types = cliParser.getOptionValues(NODE_STATE_CMD);
            if (types != null) {
                for (String type : types) {
                    if (!type.trim().isEmpty()) {
                        try {
                            nodeStates.add(NodeState.valueOf(org.apache.hadoop.util.StringUtils.toUpperCase(type.trim())));
                        } catch (IllegalArgumentException ex) {
                            sysout.println("The node state " + type + " is invalid.");
                            sysout.println(getAllValidNodeStates());
                            return exitCode;
                        }
                    }
                }
            }
        } else {
            nodeStates.add(NodeState.RUNNING);
        }
        // List all node details with more information.
        if (cliParser.hasOption(NODE_SHOW_DETAILS)) {
            listDetailedClusterNodes(nodeStates);
        } else {
            listClusterNodes(nodeStates);
        }
    } else if (cliParser.hasOption(HELP_CMD)) {
        printUsage(opts);
        return 0;
    } else {
        syserr.println("Invalid Command Usage : ");
        printUsage(opts);
    }
    return 0;
}

Example 3 with NodeState

use of org.apache.hadoop.yarn.api.records.NodeState in project hadoop by apache.

the class ClientRMService method getClusterNodes.

@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) throws YarnException {
    GetClusterNodesResponse response = recordFactory.newRecordInstance(GetClusterNodesResponse.class);
    EnumSet<NodeState> nodeStates = request.getNodeStates();
    if (nodeStates == null || nodeStates.isEmpty()) {
        nodeStates = EnumSet.allOf(NodeState.class);
    }
    Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext, nodeStates);
    List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
    for (RMNode nodeInfo : nodes) {
        nodeReports.add(createNodeReports(nodeInfo));
    }
    response.setNodeReports(nodeReports);
    return response;
}

Example 4 with NodeState

use of org.apache.hadoop.yarn.api.records.NodeState in project hadoop by apache.

the class NodesListManager method handleExcludeNodeList.

// Handle excluded nodes based on following rules:
// Recommission DECOMMISSIONED or DECOMMISSIONING nodes no longer excluded;
// Gracefully decommission excluded nodes that are not already
// DECOMMISSIONED nor DECOMMISSIONING; Take no action for excluded nodes
// that are already DECOMMISSIONED or DECOMMISSIONING.
private void handleExcludeNodeList(boolean graceful, Integer timeout) {
    // DECOMMISSIONED/DECOMMISSIONING nodes need to be re-commissioned.
    List<RMNode> nodesToRecom = new ArrayList<RMNode>();
    // Nodes need to be decommissioned (graceful or forceful);
    List<RMNode> nodesToDecom = new ArrayList<RMNode>();
    Set<String> includes = new HashSet<String>();
    Map<String, Integer> excludes = new HashMap<String, Integer>();
    hostsReader.getHostDetails(includes, excludes);
    for (RMNode n : this.rmContext.getRMNodes().values()) {
        NodeState s = n.getState();
        // An invalid node (either due to explicit exclude or not include)
        // should be excluded.
        boolean isExcluded = !isValidNode(n.getHostName(), includes, excludes.keySet());
        String nodeStr = "node " + n.getNodeID() + " with state " + s;
        if (!isExcluded) {
            // Note that no action is needed for DECOMMISSIONED node.
            if (s == NodeState.DECOMMISSIONING) {
                LOG.info("Recommission " + nodeStr);
                nodesToRecom.add(n);
            }
        // Otherwise no-action needed.
        } else {
            // exclude is true.
            if (graceful) {
                // Use per node timeout if exist otherwise the request timeout.
                Integer timeoutToUse = (excludes.get(n.getHostName()) != null) ? excludes.get(n.getHostName()) : timeout;
                if (s != NodeState.DECOMMISSIONED && s != NodeState.DECOMMISSIONING) {
                    LOG.info("Gracefully decommission " + nodeStr);
                    nodesToDecom.add(n);
                } else if (s == NodeState.DECOMMISSIONING && !Objects.equals(n.getDecommissioningTimeout(), timeoutToUse)) {
                    LOG.info("Update " + nodeStr + " timeout to be " + timeoutToUse);
                    nodesToDecom.add(n);
                } else {
                    LOG.info("No action for " + nodeStr);
                }
            } else {
                if (s != NodeState.DECOMMISSIONED) {
                    LOG.info("Forcefully decommission " + nodeStr);
                    nodesToDecom.add(n);
                }
            }
        }
    }
    for (RMNode n : nodesToRecom) {
        RMNodeEvent e = new RMNodeEvent(n.getNodeID(), RMNodeEventType.RECOMMISSION);
        this.rmContext.getDispatcher().getEventHandler().handle(e);
    }
    for (RMNode n : nodesToDecom) {
        RMNodeEvent e;
        if (graceful) {
            Integer timeoutToUse = (excludes.get(n.getHostName()) != null) ? excludes.get(n.getHostName()) : timeout;
            e = new RMNodeDecommissioningEvent(n.getNodeID(), timeoutToUse);
        } else {
            RMNodeEventType eventType = isUntrackedNode(n.getHostName()) ? RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
            e = new RMNodeEvent(n.getNodeID(), eventType);
        }
        this.rmContext.getDispatcher().getEventHandler().handle(e);
    }
    updateInactiveNodes();
}

Example 5 with NodeState

use of org.apache.hadoop.yarn.api.records.NodeState in project hadoop by apache.

the class RMWebServices method getNodes.

/**
   * Returns all nodes in the cluster. If the states param is given, returns
   * all nodes that are in the comma-separated list of states.
   */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public NodesInfo getNodes(@QueryParam("states") String states) {
    init();
    ResourceScheduler sched = this.rm.getResourceScheduler();
    if (sched == null) {
        throw new NotFoundException("Null ResourceScheduler instance");
    }
    EnumSet<NodeState> acceptedStates;
    if (states == null) {
        acceptedStates = EnumSet.allOf(NodeState.class);
    } else {
        acceptedStates = EnumSet.noneOf(NodeState.class);
        for (String stateStr : states.split(",")) {
            acceptedStates.add(NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
        }
    }
    Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(), acceptedStates);
    NodesInfo nodesInfo = new NodesInfo();
    for (RMNode rmNode : rmNodes) {
        NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
        if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED).contains(rmNode.getState())) {
            nodeInfo.setNodeHTTPAddress(EMPTY);
        }
        nodesInfo.add(nodeInfo);
    }
    return nodesInfo;
}