Examples with DatanodeDescriptor org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor used on opensource projects

Example 11 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class DFSNetworkTopology method chooseRandomWithStorageTypeAndExcludeRoot.

/**
   * Choose a random node that has the required storage type, under the given
   * root, with an excluded subtree root (could also just be a leaf node).
   *
   * Note that excludedNode is checked after a random node, so it is not being
   * handled here.
   *
   * @param root the root node where we start searching for a datanode
   * @param excludeRoot the root of the subtree what should be excluded
   * @param type the expected storage type
   * @return a random datanode, with the storage type, and is not in excluded
   * scope
   */
private Node chooseRandomWithStorageTypeAndExcludeRoot(DFSTopologyNodeImpl root, Node excludeRoot, StorageType type) {
    Node chosenNode;
    if (root.isRack()) {
        // children are datanode descriptor
        ArrayList<Node> candidates = new ArrayList<>();
        for (Node node : root.getChildren()) {
            if (node.equals(excludeRoot)) {
                continue;
            }
            DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) node;
            if (dnDescriptor.hasStorageType(type)) {
                candidates.add(node);
            }
        }
        if (candidates.size() == 0) {
            return null;
        }
        // to this point, all nodes in candidates are valid choices, and they are
        // all datanodes, pick a random one.
        chosenNode = candidates.get(RANDOM.nextInt(candidates.size()));
    } else {
        // the children are inner nodes
        ArrayList<DFSTopologyNodeImpl> candidates = getEligibleChildren(root, excludeRoot, type);
        if (candidates.size() == 0) {
            return null;
        }
        // again, all children are also inner nodes, we can do this cast.
        // to maintain uniformality, the search needs to be based on the counts
        // of valid datanodes. Below is a random weighted choose.
        int totalCounts = 0;
        int[] countArray = new int[candidates.size()];
        for (int i = 0; i < candidates.size(); i++) {
            DFSTopologyNodeImpl innerNode = candidates.get(i);
            int subTreeCount = innerNode.getSubtreeStorageCount(type);
            totalCounts += subTreeCount;
            countArray[i] = subTreeCount;
        }
        // generate a random val between [1, totalCounts]
        int randomCounts = RANDOM.nextInt(totalCounts) + 1;
        int idxChosen = 0;
        // search, but does not seem to worth it here.
        for (int i = 0; i < countArray.length; i++) {
            if (randomCounts <= countArray[i]) {
                idxChosen = i;
                break;
            }
            randomCounts -= countArray[i];
        }
        DFSTopologyNodeImpl nextRoot = candidates.get(idxChosen);
        chosenNode = chooseRandomWithStorageTypeAndExcludeRoot(nextRoot, excludeRoot, type);
    }
    return chosenNode;
}

Example 12 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class DFSNetworkTopology method chooseRandomWithStorageType.

/**
   * Choose a random node based on given scope, excludedScope and excludedNodes
   * set. Although in general the topology has at most three layers, this class
   * will not impose such assumption.
   *
   * At high level, the idea is like this, say:
   *
   * R has two children A and B, and storage type is X, say:
   * A has X = 6 (rooted at A there are 6 datanodes with X) and B has X = 8.
   *
   * Then R will generate a random int between 1~14, if it's <= 6, recursively
   * call into A, otherwise B. This will maintain a uniformed randomness of
   * choosing datanodes.
   *
   * The tricky part is how to handle excludes.
   *
   * For excludedNodes, since this set is small: currently the main reason of
   * being an excluded node is because it already has a replica. So randomly
   * picking up this node again should be rare. Thus we only check that, if the
   * chosen node is excluded, we do chooseRandom again.
   *
   * For excludedScope, we locate the root of the excluded scope. Subtracting
   * all it's ancestors' storage counters accordingly, this way the excluded
   * root is out of the picture.
   *
   * TODO : this function has duplicate code as NetworkTopology, need to
   * refactor in the future.
   *
   * @param scope
   * @param excludedScope
   * @param excludedNodes
   * @return
   */
@VisibleForTesting
Node chooseRandomWithStorageType(final String scope, String excludedScope, final Collection<Node> excludedNodes, StorageType type) {
    if (excludedScope != null) {
        if (scope.startsWith(excludedScope)) {
            return null;
        }
        if (!excludedScope.startsWith(scope)) {
            excludedScope = null;
        }
    }
    Node node = getNode(scope);
    if (node == null) {
        LOG.debug("Invalid scope {}, non-existing node", scope);
        return null;
    }
    if (!(node instanceof DFSTopologyNodeImpl)) {
        // a node is either DFSTopologyNodeImpl, or a DatanodeDescriptor
        return ((DatanodeDescriptor) node).hasStorageType(type) ? node : null;
    }
    DFSTopologyNodeImpl root = (DFSTopologyNodeImpl) node;
    Node excludeRoot = excludedScope == null ? null : getNode(excludedScope);
    // check to see if there are nodes satisfying the condition at all
    int availableCount = root.getSubtreeStorageCount(type);
    if (excludeRoot != null && root.isAncestor(excludeRoot)) {
        if (excludeRoot instanceof DFSTopologyNodeImpl) {
            availableCount -= ((DFSTopologyNodeImpl) excludeRoot).getSubtreeStorageCount(type);
        } else {
            availableCount -= ((DatanodeDescriptor) excludeRoot).hasStorageType(type) ? 1 : 0;
        }
    }
    if (excludedNodes != null) {
        for (Node excludedNode : excludedNodes) {
            // all excluded nodes should be DatanodeDescriptor
            Preconditions.checkArgument(excludedNode instanceof DatanodeDescriptor);
            availableCount -= ((DatanodeDescriptor) excludedNode).hasStorageType(type) ? 1 : 0;
        }
    }
    if (availableCount <= 0) {
        // should never be <0 in general, adding <0 check for safety purpose
        return null;
    }
    // to this point, it is guaranteed that there is at least one node
    // that satisfies the requirement, keep trying until we found one.
    Node chosen;
    do {
        chosen = chooseRandomWithStorageTypeAndExcludeRoot(root, excludeRoot, type);
        if (excludedNodes == null || !excludedNodes.contains(chosen)) {
            break;
        } else {
            LOG.debug("Node {} is excluded, continuing.", chosen);
        }
    } while (true);
    LOG.debug("chooseRandom returning {}", chosen);
    return chosen;
}

Example 13 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class DFSTopologyNodeImpl method remove.

@Override
public boolean remove(Node n) {
    if (!isAncestor(n)) {
        throw new IllegalArgumentException(n.getName() + ", which is located at " + n.getNetworkLocation() + ", is not a descendant of " + getPath(this));
    }
    // In HDFS topology, the leaf node should always be DatanodeDescriptor
    if (!(n instanceof DatanodeDescriptor)) {
        throw new IllegalArgumentException("Unexpected node type " + n.getClass().getName());
    }
    DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
    if (isParent(n)) {
        // this node is the parent of n; remove n directly
        if (childrenMap.containsKey(n.getName())) {
            for (int i = 0; i < children.size(); i++) {
                if (children.get(i).getName().equals(n.getName())) {
                    children.remove(i);
                    childrenMap.remove(n.getName());
                    childrenStorageInfo.remove(dnDescriptor.getName());
                    for (StorageType st : dnDescriptor.getStorageTypes()) {
                        decStorageTypeCount(st);
                    }
                    numOfLeaves--;
                    n.setParent(null);
                    return true;
                }
            }
        }
        return false;
    } else {
        // find the next ancestor node: the parent node
        String parentName = getNextAncestorName(n);
        DFSTopologyNodeImpl parentNode = (DFSTopologyNodeImpl) childrenMap.get(parentName);
        if (parentNode == null) {
            return false;
        }
        // remove n from the parent node
        boolean isRemoved = parentNode.remove(n);
        if (isRemoved) {
            // if the parent node has no children, remove the parent node too
            EnumMap<StorageType, Integer> currentCount = childrenStorageInfo.get(parentNode.getName());
            EnumSet<StorageType> toRemove = EnumSet.noneOf(StorageType.class);
            for (StorageType st : dnDescriptor.getStorageTypes()) {
                int newCount = currentCount.get(st) - 1;
                if (newCount == 0) {
                    toRemove.add(st);
                }
                currentCount.put(st, newCount);
            }
            for (StorageType st : toRemove) {
                currentCount.remove(st);
            }
            for (StorageType st : dnDescriptor.getStorageTypes()) {
                decStorageTypeCount(st);
            }
            if (parentNode.getNumOfChildren() == 0) {
                for (int i = 0; i < children.size(); i++) {
                    if (children.get(i).getName().equals(parentName)) {
                        children.remove(i);
                        childrenMap.remove(parentName);
                        childrenStorageInfo.remove(parentNode.getName());
                        break;
                    }
                }
            }
            numOfLeaves--;
        }
        return isRemoved;
    }
}

Example 14 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class DFSTopologyNodeImpl method add.

@Override
public boolean add(Node n) {
    if (!isAncestor(n)) {
        throw new IllegalArgumentException(n.getName() + ", which is located at " + n.getNetworkLocation() + ", is not a descendant of " + getPath(this));
    }
    // In HDFS topology, the leaf node should always be DatanodeDescriptor
    if (!(n instanceof DatanodeDescriptor)) {
        throw new IllegalArgumentException("Unexpected node type " + n.getClass().getName());
    }
    DatanodeDescriptor dnDescriptor = (DatanodeDescriptor) n;
    if (isParent(n)) {
        // this node is the parent of n; add n directly
        n.setParent(this);
        n.setLevel(this.level + 1);
        Node prev = childrenMap.put(n.getName(), n);
        if (prev != null) {
            for (int i = 0; i < children.size(); i++) {
                if (children.get(i).getName().equals(n.getName())) {
                    children.set(i, n);
                    return false;
                }
            }
        }
        children.add(n);
        numOfLeaves++;
        if (!childrenStorageInfo.containsKey(dnDescriptor.getName())) {
            childrenStorageInfo.put(dnDescriptor.getName(), new EnumMap<>(StorageType.class));
        }
        for (StorageType st : dnDescriptor.getStorageTypes()) {
            childrenStorageInfo.get(dnDescriptor.getName()).put(st, 1);
            incStorageTypeCount(st);
        }
        return true;
    } else {
        // find the next ancestor node
        String parentName = getNextAncestorName(n);
        InnerNode parentNode = (InnerNode) childrenMap.get(parentName);
        if (parentNode == null) {
            // create a new InnerNode
            parentNode = createParentNode(parentName);
            children.add(parentNode);
            childrenMap.put(parentNode.getName(), parentNode);
        }
        // add n to the subtree of the next ancestor node
        if (parentNode.add(n)) {
            numOfLeaves++;
            if (!childrenStorageInfo.containsKey(parentNode.getName())) {
                childrenStorageInfo.put(parentNode.getName(), new EnumMap<>(StorageType.class));
                for (StorageType st : dnDescriptor.getStorageTypes()) {
                    childrenStorageInfo.get(parentNode.getName()).put(st, 1);
                }
            } else {
                EnumMap<StorageType, Integer> currentCount = childrenStorageInfo.get(parentNode.getName());
                for (StorageType st : dnDescriptor.getStorageTypes()) {
                    if (currentCount.containsKey(st)) {
                        currentCount.put(st, currentCount.get(st) + 1);
                    } else {
                        currentCount.put(st, 1);
                    }
                }
            }
            for (StorageType st : dnDescriptor.getStorageTypes()) {
                incStorageTypeCount(st);
            }
            return true;
        } else {
            return false;
        }
    }
}

Example 15 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class AdminStatesBaseTest method putNodeInService.

/* Ask a specific NN to put the datanode in service and wait for it
   * to reach the NORMAL state.
   */
protected void putNodeInService(int nnIndex, DatanodeInfo outOfServiceNode) throws IOException {
    LOG.info("Putting node: " + outOfServiceNode + " in service");
    ArrayList<String> decommissionNodes = new ArrayList<>();
    Map<String, Long> maintenanceNodes = new HashMap<>();
    DatanodeManager dm = cluster.getNamesystem(nnIndex).getBlockManager().getDatanodeManager();
    List<DatanodeDescriptor> nodes = dm.getDatanodeListForReport(DatanodeReportType.ALL);
    for (DatanodeDescriptor node : nodes) {
        if (node.isMaintenance()) {
            maintenanceNodes.put(node.getName(), node.getMaintenanceExpireTimeInMS());
        } else if (node.isDecommissionInProgress() || node.isDecommissioned()) {
            decommissionNodes.add(node.getName());
        }
    }
    decommissionNodes.remove(outOfServiceNode.getName());
    maintenanceNodes.remove(outOfServiceNode.getName());
    hostsFileWriter.initOutOfServiceHosts(decommissionNodes, maintenanceNodes);
    refreshNodes(nnIndex);
    waitNodeState(outOfServiceNode, AdminStates.NORMAL);
}