Examples with NodeBase org.apache.hadoop.net.NodeBase used on opensource projects

Example 1 with NodeBase

use of org.apache.hadoop.net.NodeBase in project hadoop by apache.

the class DFSUtilClient method locatedBlocks2Locations.

/**
   * Convert a List<LocatedBlock> to BlockLocation[]
   * @param blocks A List<LocatedBlock> to be converted
   * @return converted array of BlockLocation
   */
public static BlockLocation[] locatedBlocks2Locations(List<LocatedBlock> blocks) {
    if (blocks == null) {
        return new BlockLocation[0];
    }
    int nrBlocks = blocks.size();
    BlockLocation[] blkLocations = new BlockLocation[nrBlocks];
    if (nrBlocks == 0) {
        return blkLocations;
    }
    int idx = 0;
    for (LocatedBlock blk : blocks) {
        assert idx < nrBlocks : "Incorrect index";
        DatanodeInfo[] locations = blk.getLocations();
        String[] hosts = new String[locations.length];
        String[] xferAddrs = new String[locations.length];
        String[] racks = new String[locations.length];
        for (int hCnt = 0; hCnt < locations.length; hCnt++) {
            hosts[hCnt] = locations[hCnt].getHostName();
            xferAddrs[hCnt] = locations[hCnt].getXferAddr();
            NodeBase node = new NodeBase(xferAddrs[hCnt], locations[hCnt].getNetworkLocation());
            racks[hCnt] = node.toString();
        }
        DatanodeInfo[] cachedLocations = blk.getCachedLocations();
        String[] cachedHosts = new String[cachedLocations.length];
        for (int i = 0; i < cachedLocations.length; i++) {
            cachedHosts[i] = cachedLocations[i].getHostName();
        }
        blkLocations[idx] = new BlockLocation(xferAddrs, hosts, cachedHosts, racks, blk.getStorageIDs(), blk.getStorageTypes(), blk.getStartOffset(), blk.getBlockSize(), blk.isCorrupt());
        idx++;
    }
    return blkLocations;
}

Example 2 with NodeBase

use of org.apache.hadoop.net.NodeBase in project hadoop by apache.

the class ClientContext method initTopologyResolution.

private void initTopologyResolution(Configuration config) {
    topologyResolutionEnabled = config.getBoolean(FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED, FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT);
    if (!topologyResolutionEnabled) {
        return;
    }
    DNSToSwitchMapping dnsToSwitchMapping = ReflectionUtils.newInstance(config.getClass(CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, ScriptBasedMapping.class, DNSToSwitchMapping.class), config);
    String clientHostName = NetUtils.getLocalHostname();
    List<String> nodes = new ArrayList<>();
    nodes.add(clientHostName);
    List<String> resolvedHosts = dnsToSwitchMapping.resolve(nodes);
    if (resolvedHosts != null && !resolvedHosts.isEmpty() && !resolvedHosts.get(0).equals(NetworkTopology.DEFAULT_RACK)) {
        // The client machine is able to resolve its own network location.
        this.clientNode = new NodeBase(clientHostName, resolvedHosts.get(0));
    }
}

Example 3 with NodeBase

use of org.apache.hadoop.net.NodeBase in project hadoop by apache.

the class FSDirWriteFileOp method getClientNode.

/**
   * Resolve clientmachine address to get a network location path
   */
static Node getClientNode(BlockManager bm, String clientMachine) {
    List<String> hosts = new ArrayList<>(1);
    hosts.add(clientMachine);
    List<String> rName = bm.getDatanodeManager().resolveNetworkLocation(hosts);
    Node clientNode = null;
    if (rName != null) {
        // Able to resolve clientMachine mapping.
        // Create a temp node to findout the rack local nodes
        clientNode = new NodeBase(rName.get(0) + NodeBase.PATH_SEPARATOR_STR + clientMachine);
    }
    return clientNode;
}

Example 4 with NodeBase

use of org.apache.hadoop.net.NodeBase in project hadoop by apache.

the class FileInputFormat method getSplitHostsAndCachedHosts.

/** 
   * This function identifies and returns the hosts that contribute 
   * most for a given split. For calculating the contribution, rack
   * locality is treated on par with host locality, so hosts from racks
   * that contribute the most are preferred over hosts on racks that 
   * contribute less
   * @param blkLocations The list of block locations
   * @param offset 
   * @param splitSize 
   * @return two arrays - one of hosts that contribute most to this split, and
   *    one of hosts that contribute most to this split that have the data
   *    cached on them
   * @throws IOException
   */
private String[][] getSplitHostsAndCachedHosts(BlockLocation[] blkLocations, long offset, long splitSize, NetworkTopology clusterMap) throws IOException {
    int startIndex = getBlockIndex(blkLocations, offset);
    long bytesInThisBlock = blkLocations[startIndex].getOffset() + blkLocations[startIndex].getLength() - offset;
    //If this is the only block, just return
    if (bytesInThisBlock >= splitSize) {
        return new String[][] { blkLocations[startIndex].getHosts(), blkLocations[startIndex].getCachedHosts() };
    }
    long bytesInFirstBlock = bytesInThisBlock;
    int index = startIndex + 1;
    splitSize -= bytesInThisBlock;
    while (splitSize > 0) {
        bytesInThisBlock = Math.min(splitSize, blkLocations[index++].getLength());
        splitSize -= bytesInThisBlock;
    }
    long bytesInLastBlock = bytesInThisBlock;
    int endIndex = index - 1;
    Map<Node, NodeInfo> hostsMap = new IdentityHashMap<Node, NodeInfo>();
    Map<Node, NodeInfo> racksMap = new IdentityHashMap<Node, NodeInfo>();
    String[] allTopos = new String[0];
    for (index = startIndex; index <= endIndex; index++) {
        // Establish the bytes in this block
        if (index == startIndex) {
            bytesInThisBlock = bytesInFirstBlock;
        } else if (index == endIndex) {
            bytesInThisBlock = bytesInLastBlock;
        } else {
            bytesInThisBlock = blkLocations[index].getLength();
        }
        allTopos = blkLocations[index].getTopologyPaths();
        // prefix a fakeRack
        if (allTopos.length == 0) {
            allTopos = fakeRacks(blkLocations, index);
        }
        for (String topo : allTopos) {
            Node node, parentNode;
            NodeInfo nodeInfo, parentNodeInfo;
            node = clusterMap.getNode(topo);
            if (node == null) {
                node = new NodeBase(topo);
                clusterMap.add(node);
            }
            nodeInfo = hostsMap.get(node);
            if (nodeInfo == null) {
                nodeInfo = new NodeInfo(node);
                hostsMap.put(node, nodeInfo);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
                if (parentNodeInfo == null) {
                    parentNodeInfo = new NodeInfo(parentNode);
                    racksMap.put(parentNode, parentNodeInfo);
                }
                parentNodeInfo.addLeaf(nodeInfo);
            } else {
                nodeInfo = hostsMap.get(node);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
            }
            nodeInfo.addValue(index, bytesInThisBlock);
            parentNodeInfo.addValue(index, bytesInThisBlock);
        }
    // for all topos
    }
    // We don't yet support cached hosts when bytesInThisBlock > splitSize
    return new String[][] { identifyHosts(allTopos.length, racksMap), new String[0] };
}