Examples with StoredReplicaState org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas.StoredReplicaState used on opensource projects

Example 1 with StoredReplicaState

use of org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas.StoredReplicaState in project hadoop by apache.

the class BlockManager method countReplicasForStripedBlock.

/**
   * For a striped block, it is possible it contains full number of internal
   * blocks (i.e., 9 by default), but with duplicated replicas of the same
   * internal block. E.g., for the following list of internal blocks
   * b0, b0, b1, b2, b3, b4, b5, b6, b7
   * we have 9 internal blocks but we actually miss b8.
   * We should use this method to detect the above scenario and schedule
   * necessary reconstruction.
   */
private void countReplicasForStripedBlock(NumberReplicas counters, BlockInfoStriped block, Collection<DatanodeDescriptor> nodesCorrupt, boolean inStartupSafeMode) {
    BitSet bitSet = new BitSet(block.getTotalBlockNum());
    for (StorageAndBlockIndex si : block.getStorageAndIndexInfos()) {
        StoredReplicaState state = checkReplicaOnStorage(counters, block, si.getStorage(), nodesCorrupt, inStartupSafeMode);
        if (state == StoredReplicaState.LIVE) {
            if (!bitSet.get(si.getBlockIndex())) {
                bitSet.set(si.getBlockIndex());
            } else {
                counters.subtract(StoredReplicaState.LIVE, 1);
                counters.add(StoredReplicaState.REDUNDANT, 1);
            }
        }
    }
}

Example 2 with StoredReplicaState

use of org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas.StoredReplicaState in project hadoop by apache.

the class BlockManager method chooseSourceDatanodes.

/**
   * Parse the data-nodes the block belongs to and choose a certain number
   * from them to be the recovery sources.
   *
   * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
   * since the former do not have write traffic and hence are less busy.
   * We do not use already decommissioned nodes as a source.
   * Otherwise we randomly choose nodes among those that did not reach their
   * replication limits. However, if the recovery work is of the highest
   * priority and all nodes have reached their replication limits, we will
   * randomly choose the desired number of nodes despite the replication limit.
   *
   * In addition form a list of all nodes containing the block
   * and calculate its replication numbers.
   *
   * @param block Block for which a replication source is needed
   * @param containingNodes List to be populated with nodes found to contain
   *                        the given block
   * @param nodesContainingLiveReplicas List to be populated with nodes found
   *                                    to contain live replicas of the given
   *                                    block
   * @param numReplicas NumberReplicas instance to be initialized with the
   *                    counts of live, corrupt, excess, and decommissioned
   *                    replicas of the given block.
   * @param liveBlockIndices List to be populated with indices of healthy
   *                         blocks in a striped block group
   * @param priority integer representing replication priority of the given
   *                 block
   * @return the array of DatanodeDescriptor of the chosen nodes from which to
   *         recover the given block
   */
@VisibleForTesting
DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block, List<DatanodeDescriptor> containingNodes, List<DatanodeStorageInfo> nodesContainingLiveReplicas, NumberReplicas numReplicas, List<Byte> liveBlockIndices, int priority) {
    containingNodes.clear();
    nodesContainingLiveReplicas.clear();
    List<DatanodeDescriptor> srcNodes = new ArrayList<>();
    liveBlockIndices.clear();
    final boolean isStriped = block.isStriped();
    BitSet bitSet = isStriped ? new BitSet(((BlockInfoStriped) block).getTotalBlockNum()) : null;
    for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
        final DatanodeDescriptor node = storage.getDatanodeDescriptor();
        final StoredReplicaState state = checkReplicaOnStorage(numReplicas, block, storage, corruptReplicas.getNodes(block), false);
        if (state == StoredReplicaState.LIVE) {
            nodesContainingLiveReplicas.add(storage);
        }
        containingNodes.add(node);
        // do not select the replica if it is corrupt or excess
        if (state == StoredReplicaState.CORRUPT || state == StoredReplicaState.EXCESS) {
            continue;
        }
        // suitable for read or unknown state replicas.
        if (state == null || state == StoredReplicaState.DECOMMISSIONED || state == StoredReplicaState.MAINTENANCE_NOT_FOR_READ) {
            continue;
        }
        if (priority != LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY && (!node.isDecommissionInProgress() && !node.isEnteringMaintenance()) && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams) {
            // already reached replication limit
            continue;
        }
        if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit) {
            continue;
        }
        if (isStriped || srcNodes.isEmpty()) {
            srcNodes.add(node);
            if (isStriped) {
                byte blockIndex = ((BlockInfoStriped) block).getStorageBlockIndex(storage);
                liveBlockIndices.add(blockIndex);
                if (!bitSet.get(blockIndex)) {
                    bitSet.set(blockIndex);
                } else if (state == StoredReplicaState.LIVE) {
                    numReplicas.subtract(StoredReplicaState.LIVE, 1);
                    numReplicas.add(StoredReplicaState.REDUNDANT, 1);
                }
            }
            continue;
        }
        // if the node failed to replicate the block on previous iterations
        if (ThreadLocalRandom.current().nextBoolean()) {
            srcNodes.set(0, node);
        }
    }
    return srcNodes.toArray(new DatanodeDescriptor[srcNodes.size()]);
}

Example 3 with StoredReplicaState

use of org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas.StoredReplicaState in project hadoop by apache.

the class BlockManager method checkReplicaOnStorage.

private StoredReplicaState checkReplicaOnStorage(NumberReplicas counters, BlockInfo b, DatanodeStorageInfo storage, Collection<DatanodeDescriptor> nodesCorrupt, boolean inStartupSafeMode) {
    final StoredReplicaState s;
    if (storage.getState() == State.NORMAL) {
        final DatanodeDescriptor node = storage.getDatanodeDescriptor();
        if (nodesCorrupt != null && nodesCorrupt.contains(node)) {
            s = StoredReplicaState.CORRUPT;
        } else if (inStartupSafeMode) {
            s = StoredReplicaState.LIVE;
            counters.add(s, 1);
            return s;
        } else if (node.isDecommissionInProgress()) {
            s = StoredReplicaState.DECOMMISSIONING;
        } else if (node.isDecommissioned()) {
            s = StoredReplicaState.DECOMMISSIONED;
        } else if (node.isMaintenance()) {
            if (node.isInMaintenance() || !node.isAlive()) {
                s = StoredReplicaState.MAINTENANCE_NOT_FOR_READ;
            } else {
                s = StoredReplicaState.MAINTENANCE_FOR_READ;
            }
        } else if (isExcess(node, b)) {
            s = StoredReplicaState.EXCESS;
        } else {
            s = StoredReplicaState.LIVE;
        }
        counters.add(s, 1);
        if (storage.areBlockContentsStale()) {
            counters.add(StoredReplicaState.STALESTORAGE, 1);
        }
    } else if (!inStartupSafeMode && storage.getState() == State.READ_ONLY_SHARED) {
        s = StoredReplicaState.READONLY;
        counters.add(s, 1);
    } else {
        s = null;
    }
    return s;
}