Examples with StorageGroup org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup used on opensource projects

Example 1 with StorageGroup

use of org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup in project hadoop by apache.

the class Balancer method init.

/**
   * Given a datanode storage set, build a network topology and decide
   * over-utilized storages, above average utilized storages, 
   * below average utilized storages, and underutilized storages. 
   * The input datanode storage set is shuffled in order to randomize
   * to the storage matching later on.
   *
   * @return the number of bytes needed to move in order to balance the cluster.
   */
private long init(List<DatanodeStorageReport> reports) {
    // compute average utilization
    for (DatanodeStorageReport r : reports) {
        policy.accumulateSpaces(r);
    }
    policy.initAvgUtilization();
    // create network topology and classify utilization collections: 
    //   over-utilized, above-average, below-average and under-utilized.
    long overLoadedBytes = 0L, underLoadedBytes = 0L;
    for (DatanodeStorageReport r : reports) {
        final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
        final boolean isSource = Util.isIncluded(sourceNodes, dn.getDatanodeInfo());
        for (StorageType t : StorageType.getMovableTypes()) {
            final Double utilization = policy.getUtilization(r, t);
            if (utilization == null) {
                // datanode does not have such storage type 
                continue;
            }
            final double average = policy.getAvgUtilization(t);
            if (utilization >= average && !isSource) {
                LOG.info(dn + "[" + t + "] has utilization=" + utilization + " >= average=" + average + " but it is not specified as a source; skipping it.");
                continue;
            }
            final double utilizationDiff = utilization - average;
            final long capacity = getCapacity(r, t);
            final double thresholdDiff = Math.abs(utilizationDiff) - threshold;
            final long maxSize2Move = computeMaxSize2Move(capacity, getRemaining(r, t), utilizationDiff, maxSizeToMove);
            final StorageGroup g;
            if (utilizationDiff > 0) {
                final Source s = dn.addSource(t, maxSize2Move, dispatcher);
                if (thresholdDiff <= 0) {
                    // within threshold
                    aboveAvgUtilized.add(s);
                } else {
                    overLoadedBytes += percentage2bytes(thresholdDiff, capacity);
                    overUtilized.add(s);
                }
                g = s;
            } else {
                g = dn.addTarget(t, maxSize2Move);
                if (thresholdDiff <= 0) {
                    // within threshold
                    belowAvgUtilized.add(g);
                } else {
                    underLoadedBytes += percentage2bytes(thresholdDiff, capacity);
                    underUtilized.add(g);
                }
            }
            dispatcher.getStorageGroupMap().put(g);
        }
    }
    logUtilizationCollections();
    Preconditions.checkState(dispatcher.getStorageGroupMap().size() == overUtilized.size() + underUtilized.size() + aboveAvgUtilized.size() + belowAvgUtilized.size(), "Mismatched number of storage groups");
    // return number of bytes to be moved in order to make the cluster balanced
    return Math.max(overLoadedBytes, underLoadedBytes);
}

Example 2 with StorageGroup

use of org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup in project hadoop by apache.

the class Dispatcher method checkForBlockPinningFailures.

/**
   * Check any of the block movements are failed due to block pinning errors. If
   * yes, add the failed blockId and its respective source node location to the
   * excluded list.
   */
public static void checkForBlockPinningFailures(Map<Long, Set<DatanodeInfo>> excludedPinnedBlocks, Iterable<? extends StorageGroup> targets) {
    for (StorageGroup t : targets) {
        Map<Long, Set<DatanodeInfo>> blockPinningFailureList = t.getDDatanode().getBlockPinningFailureList();
        Set<Entry<Long, Set<DatanodeInfo>>> entrySet = blockPinningFailureList.entrySet();
        for (Entry<Long, Set<DatanodeInfo>> entry : entrySet) {
            Long blockId = entry.getKey();
            Set<DatanodeInfo> locs = excludedPinnedBlocks.get(blockId);
            if (locs == null) {
                // blockId doesn't exists in the excluded list.
                locs = entry.getValue();
                excludedPinnedBlocks.put(blockId, locs);
            } else {
                // blockId already exists in the excluded list, add the pinned node.
                locs.addAll(entry.getValue());
            }
        }
    }
}

Example 3 with StorageGroup

use of org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup in project hadoop by apache.

the class Dispatcher method reset.

/** Reset all fields in order to prepare for the next iteration */
void reset(Configuration conf) {
    cluster = NetworkTopology.getInstance(conf);
    storageGroupMap.clear();
    sources.clear();
    moverThreadAllocator.reset();
    for (StorageGroup t : targets) {
        t.getDDatanode().shutdownMoveExecutor();
    }
    targets.clear();
    globalBlocks.removeAllButRetain(movedBlocks);
    movedBlocks.cleanup();
}

Example 4 with StorageGroup

use of org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup in project hadoop by apache.

the class Mover method init.

void init() throws IOException {
    initStoragePolicies();
    final List<DatanodeStorageReport> reports = dispatcher.init();
    for (DatanodeStorageReport r : reports) {
        final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo());
        for (StorageType t : StorageType.getMovableTypes()) {
            final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher);
            final long maxRemaining = getMaxRemaining(r, t);
            final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t, maxRemaining) : null;
            storages.add(source, target);
        }
    }
}

Example 5 with StorageGroup

use of org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup in project hadoop by apache.

the class Mover method newDBlock.

DBlock newDBlock(LocatedBlock lb, List<MLocation> locations, ErasureCodingPolicy ecPolicy) {
    Block blk = lb.getBlock().getLocalBlock();
    DBlock db;
    if (lb.isStriped()) {
        LocatedStripedBlock lsb = (LocatedStripedBlock) lb;
        byte[] indices = new byte[lsb.getBlockIndices().length];
        for (int i = 0; i < indices.length; i++) {
            indices[i] = (byte) lsb.getBlockIndices()[i];
        }
        db = new DBlockStriped(blk, indices, (short) ecPolicy.getNumDataUnits(), ecPolicy.getCellSize());
    } else {
        db = new DBlock(blk);
    }
    for (MLocation ml : locations) {
        StorageGroup source = storages.getSource(ml);
        if (source != null) {
            db.addLocation(source);
        }
    }
    return db;
}