Examples with VisibleForTesting com.google.common.annotations.VisibleForTesting used on opensource projects

Example 26 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class KeyProviderCache method setKeyProvider.

@VisibleForTesting
public void setKeyProvider(Configuration conf, KeyProvider keyProvider) {
    URI uri = createKeyProviderURI(conf);
    assert uri != null;
    cache.put(uri, keyProvider);
}

Example 27 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class DataNode method transferBlock.

@VisibleForTesting
void transferBlock(ExtendedBlock block, DatanodeInfo[] xferTargets, StorageType[] xferTargetStorageTypes) throws IOException {
    BPOfferService bpos = getBPOSForBlock(block);
    DatanodeRegistration bpReg = getDNRegistrationForBP(block.getBlockPoolId());
    boolean replicaNotExist = false;
    boolean replicaStateNotFinalized = false;
    boolean blockFileNotExist = false;
    boolean lengthTooShort = false;
    try {
        data.checkBlock(block, block.getNumBytes(), ReplicaState.FINALIZED);
    } catch (ReplicaNotFoundException e) {
        replicaNotExist = true;
    } catch (UnexpectedReplicaStateException e) {
        replicaStateNotFinalized = true;
    } catch (FileNotFoundException e) {
        blockFileNotExist = true;
    } catch (EOFException e) {
        lengthTooShort = true;
    } catch (IOException e) {
        // The IOException indicates not being able to access block file,
        // treat it the same here as blockFileNotExist, to trigger 
        // reporting it as a bad block
        blockFileNotExist = true;
    }
    if (replicaNotExist || replicaStateNotFinalized) {
        String errStr = "Can't send invalid block " + block;
        LOG.info(errStr);
        bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errStr);
        return;
    }
    if (blockFileNotExist) {
        // Report back to NN bad block caused by non-existent block file.
        reportBadBlock(bpos, block, "Can't replicate block " + block + " because the block file doesn't exist, or is not accessible");
        return;
    }
    if (lengthTooShort) {
        // Check if NN recorded length matches on-disk length 
        // Shorter on-disk len indicates corruption so report NN the corrupt block
        reportBadBlock(bpos, block, "Can't replicate block " + block + " because on-disk length " + data.getLength(block) + " is shorter than NameNode recorded length " + block.getNumBytes());
        return;
    }
    int numTargets = xferTargets.length;
    if (numTargets > 0) {
        StringBuilder xfersBuilder = new StringBuilder();
        for (int i = 0; i < numTargets; i++) {
            xfersBuilder.append(xferTargets[i]);
            xfersBuilder.append(" ");
        }
        LOG.info(bpReg + " Starting thread to transfer " + block + " to " + xfersBuilder);
        new Daemon(new DataTransfer(xferTargets, xferTargetStorageTypes, block, BlockConstructionStage.PIPELINE_SETUP_CREATE, "")).start();
    }
}

Example 28 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class BlockManager method computeReconstructionWorkForBlocks.

/**
   * Reconstruct a set of blocks to full strength through replication or
   * erasure coding
   *
   * @param blocksToReconstruct blocks to be reconstructed, for each priority
   * @return the number of blocks scheduled for replication
   */
@VisibleForTesting
int computeReconstructionWorkForBlocks(List<List<BlockInfo>> blocksToReconstruct) {
    int scheduledWork = 0;
    List<BlockReconstructionWork> reconWork = new LinkedList<>();
    // Step 1: categorize at-risk blocks into replication and EC tasks
    namesystem.writeLock();
    try {
        synchronized (neededReconstruction) {
            for (int priority = 0; priority < blocksToReconstruct.size(); priority++) {
                for (BlockInfo block : blocksToReconstruct.get(priority)) {
                    BlockReconstructionWork rw = scheduleReconstruction(block, priority);
                    if (rw != null) {
                        reconWork.add(rw);
                    }
                }
            }
        }
    } finally {
        namesystem.writeUnlock();
    }
    // Step 2: choose target nodes for each reconstruction task
    final Set<Node> excludedNodes = new HashSet<>();
    for (BlockReconstructionWork rw : reconWork) {
        // Exclude all of the containing nodes from being targets.
        // This list includes decommissioning or corrupt nodes.
        excludedNodes.clear();
        for (DatanodeDescriptor dn : rw.getContainingNodes()) {
            excludedNodes.add(dn);
        }
        // choose replication targets: NOT HOLDING THE GLOBAL LOCK
        // It is costly to extract the filename for which chooseTargets is called,
        // so for now we pass in the block collection itself.
        final BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(rw.getBlock().getBlockType());
        rw.chooseTargets(placementPolicy, storagePolicySuite, excludedNodes);
    }
    // Step 3: add tasks to the DN
    namesystem.writeLock();
    try {
        for (BlockReconstructionWork rw : reconWork) {
            final DatanodeStorageInfo[] targets = rw.getTargets();
            if (targets == null || targets.length == 0) {
                rw.resetTargets();
                continue;
            }
            synchronized (neededReconstruction) {
                if (validateReconstructionWork(rw)) {
                    scheduledWork++;
                }
            }
        }
    } finally {
        namesystem.writeUnlock();
    }
    if (blockLog.isDebugEnabled()) {
        // log which blocks have been scheduled for reconstruction
        for (BlockReconstructionWork rw : reconWork) {
            DatanodeStorageInfo[] targets = rw.getTargets();
            if (targets != null && targets.length != 0) {
                StringBuilder targetList = new StringBuilder("datanode(s)");
                for (DatanodeStorageInfo target : targets) {
                    targetList.append(' ');
                    targetList.append(target.getDatanodeDescriptor());
                }
                blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.getSrcNodes(), rw.getBlock(), targetList);
            }
        }
        blockLog.debug("BLOCK* neededReconstruction = {} pendingReconstruction = {}", neededReconstruction.size(), pendingReconstruction.size());
    }
    return scheduledWork;
}

Example 29 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class BlockManager method chooseSourceDatanodes.

/**
   * Parse the data-nodes the block belongs to and choose a certain number
   * from them to be the recovery sources.
   *
   * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
   * since the former do not have write traffic and hence are less busy.
   * We do not use already decommissioned nodes as a source.
   * Otherwise we randomly choose nodes among those that did not reach their
   * replication limits. However, if the recovery work is of the highest
   * priority and all nodes have reached their replication limits, we will
   * randomly choose the desired number of nodes despite the replication limit.
   *
   * In addition form a list of all nodes containing the block
   * and calculate its replication numbers.
   *
   * @param block Block for which a replication source is needed
   * @param containingNodes List to be populated with nodes found to contain
   *                        the given block
   * @param nodesContainingLiveReplicas List to be populated with nodes found
   *                                    to contain live replicas of the given
   *                                    block
   * @param numReplicas NumberReplicas instance to be initialized with the
   *                    counts of live, corrupt, excess, and decommissioned
   *                    replicas of the given block.
   * @param liveBlockIndices List to be populated with indices of healthy
   *                         blocks in a striped block group
   * @param priority integer representing replication priority of the given
   *                 block
   * @return the array of DatanodeDescriptor of the chosen nodes from which to
   *         recover the given block
   */
@VisibleForTesting
DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block, List<DatanodeDescriptor> containingNodes, List<DatanodeStorageInfo> nodesContainingLiveReplicas, NumberReplicas numReplicas, List<Byte> liveBlockIndices, int priority) {
    containingNodes.clear();
    nodesContainingLiveReplicas.clear();
    List<DatanodeDescriptor> srcNodes = new ArrayList<>();
    liveBlockIndices.clear();
    final boolean isStriped = block.isStriped();
    BitSet bitSet = isStriped ? new BitSet(((BlockInfoStriped) block).getTotalBlockNum()) : null;
    for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
        final DatanodeDescriptor node = storage.getDatanodeDescriptor();
        final StoredReplicaState state = checkReplicaOnStorage(numReplicas, block, storage, corruptReplicas.getNodes(block), false);
        if (state == StoredReplicaState.LIVE) {
            nodesContainingLiveReplicas.add(storage);
        }
        containingNodes.add(node);
        // do not select the replica if it is corrupt or excess
        if (state == StoredReplicaState.CORRUPT || state == StoredReplicaState.EXCESS) {
            continue;
        }
        // suitable for read or unknown state replicas.
        if (state == null || state == StoredReplicaState.DECOMMISSIONED || state == StoredReplicaState.MAINTENANCE_NOT_FOR_READ) {
            continue;
        }
        if (priority != LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY && (!node.isDecommissionInProgress() && !node.isEnteringMaintenance()) && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams) {
            // already reached replication limit
            continue;
        }
        if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit) {
            continue;
        }
        if (isStriped || srcNodes.isEmpty()) {
            srcNodes.add(node);
            if (isStriped) {
                byte blockIndex = ((BlockInfoStriped) block).getStorageBlockIndex(storage);
                liveBlockIndices.add(blockIndex);
                if (!bitSet.get(blockIndex)) {
                    bitSet.set(blockIndex);
                } else if (state == StoredReplicaState.LIVE) {
                    numReplicas.subtract(StoredReplicaState.LIVE, 1);
                    numReplicas.add(StoredReplicaState.REDUNDANT, 1);
                }
            }
            continue;
        }
        // if the node failed to replicate the block on previous iterations
        if (ThreadLocalRandom.current().nextBoolean()) {
            srcNodes.set(0, node);
        }
    }
    return srcNodes.toArray(new DatanodeDescriptor[srcNodes.size()]);
}

Example 30 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class BlockPoolSliceStorage method getRestoreDirectory.

/**
   * Get a target subdirectory under current/ for a given block file that is being
   * restored from trash.
   *
   * The subdirectory structure under trash/ mirrors that under current/ to keep
   * implicit memory of where the files are to be restored.
   *
   * @return the target directory to restore a previously deleted block file.
   */
@VisibleForTesting
String getRestoreDirectory(File blockFile) {
    Matcher matcher = BLOCK_POOL_TRASH_PATH_PATTERN.matcher(blockFile.getParent());
    String restoreDirectory = matcher.replaceFirst("$1$2" + STORAGE_DIR_CURRENT + "$4");
    LOG.info("Restoring " + blockFile + " to " + restoreDirectory);
    return restoreDirectory;
}