Search in sources :

Example 1 with HadoopIgfsEndpoint

use of org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint in project ignite by apache.

the class IgniteHadoopWeightedMapReducePlanner method igfsAffinityNodesForSplit.

/**
 * Get IGFS affinity nodes for split if possible.
 * <p>
 * Order in the returned collection *is* significant, meaning that nodes containing more data
 * go first. This way, the 1st nodes in the collection considered to be preferable for scheduling.
 *
 * @param split Input split.
 * @return IGFS affinity or {@code null} if IGFS is not available.
 * @throws IgniteCheckedException If failed.
 */
@Nullable
private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
    if (split instanceof HadoopFileBlock) {
        HadoopFileBlock split0 = (HadoopFileBlock) split;
        if (IgniteFileSystem.IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
            HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
            IgfsEx igfs = (IgfsEx) ((IgniteEx) ignite).igfsx(endpoint.igfs());
            if (igfs != null && !igfs.isProxy(split0.file())) {
                IgfsPath path = new IgfsPath(split0.file());
                if (igfs.exists(path)) {
                    Collection<IgfsBlockLocation> blocks;
                    try {
                        blocks = igfs.affinity(path, split0.start(), split0.length());
                    } catch (IgniteException e) {
                        throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path + ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
                    }
                    assert blocks != null;
                    if (blocks.size() == 1)
                        return blocks.iterator().next().nodeIds();
                    else {
                        // The most "local" nodes go first.
                        Map<UUID, Long> idToLen = new HashMap<>();
                        for (IgfsBlockLocation block : blocks) {
                            for (UUID id : block.nodeIds()) {
                                Long len = idToLen.get(id);
                                idToLen.put(id, len == null ? block.length() : block.length() + len);
                            }
                        }
                        // Sort the nodes in non-ascending order by contained data lengths.
                        Map<NodeIdAndLength, UUID> res = new TreeMap<>();
                        for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
                            UUID id = idToLenEntry.getKey();
                            res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
                        }
                        return new LinkedHashSet<>(res.values());
                    }
                }
            }
        }
    }
    return null;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) HashMap(java.util.HashMap) IdentityHashMap(java.util.IdentityHashMap) HadoopIgfsEndpoint(org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint) IgfsBlockLocation(org.apache.ignite.igfs.IgfsBlockLocation) HadoopFileBlock(org.apache.ignite.internal.processors.hadoop.HadoopFileBlock) TreeMap(java.util.TreeMap) IgfsPath(org.apache.ignite.igfs.IgfsPath) IgfsEx(org.apache.ignite.internal.processors.igfs.IgfsEx) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) UUID(java.util.UUID) HashMap(java.util.HashMap) Map(java.util.Map) IdentityHashMap(java.util.IdentityHashMap) TreeMap(java.util.TreeMap) Nullable(org.jetbrains.annotations.Nullable)

Aggregations

HashMap (java.util.HashMap)1 IdentityHashMap (java.util.IdentityHashMap)1 LinkedHashSet (java.util.LinkedHashSet)1 Map (java.util.Map)1 TreeMap (java.util.TreeMap)1 UUID (java.util.UUID)1 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)1 IgniteException (org.apache.ignite.IgniteException)1 IgfsBlockLocation (org.apache.ignite.igfs.IgfsBlockLocation)1 IgfsPath (org.apache.ignite.igfs.IgfsPath)1 HadoopFileBlock (org.apache.ignite.internal.processors.hadoop.HadoopFileBlock)1 HadoopIgfsEndpoint (org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint)1 IgfsEx (org.apache.ignite.internal.processors.igfs.IgfsEx)1 Nullable (org.jetbrains.annotations.Nullable)1