Examples with BlockUCState org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState used on opensource projects

Example 1 with BlockUCState

use of org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState in project hadoop by apache.

the class BlockManager method processFirstBlockReport.

/**
   * processFirstBlockReport is intended only for processing "initial" block
   * reports, the first block report received from a DN after it registers.
   * It just adds all the valid replicas to the datanode, without calculating 
   * a toRemove list (since there won't be any).  It also silently discards 
   * any invalid blocks, thereby deferring their processing until 
   * the next block report.
   * @param storageInfo - DatanodeStorageInfo that sent the report
   * @param report - the initial block report, to be processed
   * @throws IOException 
   */
private void processFirstBlockReport(final DatanodeStorageInfo storageInfo, final BlockListAsLongs report) throws IOException {
    if (report == null)
        return;
    assert (namesystem.hasWriteLock());
    assert (storageInfo.getBlockReportCount() == 0);
    for (BlockReportReplica iblk : report) {
        ReplicaState reportedState = iblk.getState();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Initial report of block " + iblk.getBlockName() + " on " + storageInfo.getDatanodeDescriptor() + " size " + iblk.getNumBytes() + " replicaState = " + reportedState);
        }
        if (shouldPostponeBlocksFromFuture && isGenStampInFuture(iblk)) {
            queueReportedBlock(storageInfo, iblk, reportedState, QUEUE_REASON_FUTURE_GENSTAMP);
            continue;
        }
        BlockInfo storedBlock = getStoredBlock(iblk);
        // an integrity assumption of Name node
        if (storedBlock == null) {
            bmSafeMode.checkBlocksWithFutureGS(iblk);
            continue;
        }
        // If block is corrupt, mark it and continue to next block.
        BlockUCState ucState = storedBlock.getBlockUCState();
        BlockToMarkCorrupt c = checkReplicaCorrupt(iblk, reportedState, storedBlock, ucState, storageInfo.getDatanodeDescriptor());
        if (c != null) {
            if (shouldPostponeBlocksFromFuture) {
                // In the Standby, we may receive a block report for a file that we
                // just have an out-of-date gen-stamp or state for, for example.
                queueReportedBlock(storageInfo, iblk, reportedState, QUEUE_REASON_CORRUPT_STATE);
            } else {
                markBlockAsCorrupt(c, storageInfo, storageInfo.getDatanodeDescriptor());
            }
            continue;
        }
        // If block is under construction, add this replica to its list
        if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
            storedBlock.getUnderConstructionFeature().addReplicaIfNotPresent(storageInfo, iblk, reportedState);
            // refer HDFS-5283
            if (namesystem.isInSnapshot(storedBlock.getBlockCollectionId())) {
                int numOfReplicas = storedBlock.getUnderConstructionFeature().getNumExpectedLocations();
                bmSafeMode.incrementSafeBlockCount(numOfReplicas, storedBlock);
            }
        //and fall through to next clause
        }
        //add replica if appropriate
        if (reportedState == ReplicaState.FINALIZED) {
            addStoredBlockImmediate(storedBlock, iblk, storageInfo);
        }
    }
}

Example 2 with BlockUCState

use of org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState in project hadoop by apache.

the class BlockManager method reportDiffSortedInner.

private void reportDiffSortedInner(final DatanodeStorageInfo storageInfo, final BlockReportReplica replica, final ReplicaState reportedState, final BlockInfo storedBlock, final Collection<BlockInfoToAdd> toAdd, final Collection<BlockToMarkCorrupt> toCorrupt, final Collection<StatefulBlockInfo> toUC) {
    assert replica != null;
    assert storedBlock != null;
    DatanodeDescriptor dn = storageInfo.getDatanodeDescriptor();
    BlockUCState ucState = storedBlock.getBlockUCState();
    // Block is on the NN
    if (LOG.isDebugEnabled()) {
        LOG.debug("In memory blockUCState = " + ucState);
    }
    // Ignore replicas already scheduled to be removed from the DN
    if (invalidateBlocks.contains(dn, replica)) {
        return;
    }
    BlockToMarkCorrupt c = checkReplicaCorrupt(replica, reportedState, storedBlock, ucState, dn);
    if (c != null) {
        if (shouldPostponeBlocksFromFuture) {
            // If the block is an out-of-date generation stamp or state,
            // but we're the standby, we shouldn't treat it as corrupt,
            // but instead just queue it for later processing.
            // TODO: Pretty confident this should be s/storedBlock/block below,
            // since we should be postponing the info of the reported block, not
            // the stored block. See HDFS-6289 for more context.
            queueReportedBlock(storageInfo, storedBlock, reportedState, QUEUE_REASON_CORRUPT_STATE);
        } else {
            toCorrupt.add(c);
        }
    } else if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
        toUC.add(new StatefulBlockInfo(storedBlock, new Block(replica), reportedState));
    } else if (reportedState == ReplicaState.FINALIZED && (storedBlock.findStorageInfo(storageInfo) == -1 || corruptReplicas.isReplicaCorrupt(storedBlock, dn))) {
        // Add replica if appropriate. If the replica was previously corrupt
        // but now okay, it might need to be updated.
        toAdd.add(new BlockInfoToAdd(storedBlock, new Block(replica)));
    }
}

Example 3 with BlockUCState

use of org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState in project hadoop by apache.

the class BlockManager method processAndHandleReportedBlock.

private void processAndHandleReportedBlock(DatanodeStorageInfo storageInfo, Block block, ReplicaState reportedState, DatanodeDescriptor delHintNode) throws IOException {
    final DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Reported block " + block + " on " + node + " size " + block.getNumBytes() + " replicaState = " + reportedState);
    }
    if (shouldPostponeBlocksFromFuture && isGenStampInFuture(block)) {
        queueReportedBlock(storageInfo, block, reportedState, QUEUE_REASON_FUTURE_GENSTAMP);
        return;
    }
    // find block by blockId
    BlockInfo storedBlock = getStoredBlock(block);
    if (storedBlock == null) {
        // If blocksMap does not contain reported block id,
        // the replica should be removed from the data-node.
        blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not " + "belong to any file", block, node, block.getNumBytes());
        addToInvalidates(new Block(block), node);
        return;
    }
    BlockUCState ucState = storedBlock.getBlockUCState();
    // Block is on the NN
    if (LOG.isDebugEnabled()) {
        LOG.debug("In memory blockUCState = " + ucState);
    }
    // Ignore replicas already scheduled to be removed from the DN
    if (invalidateBlocks.contains(node, block)) {
        return;
    }
    BlockToMarkCorrupt c = checkReplicaCorrupt(block, reportedState, storedBlock, ucState, node);
    if (c != null) {
        if (shouldPostponeBlocksFromFuture) {
            // If the block is an out-of-date generation stamp or state,
            // but we're the standby, we shouldn't treat it as corrupt,
            // but instead just queue it for later processing.
            // TODO: Pretty confident this should be s/storedBlock/block below,
            // since we should be postponing the info of the reported block, not
            // the stored block. See HDFS-6289 for more context.
            queueReportedBlock(storageInfo, storedBlock, reportedState, QUEUE_REASON_CORRUPT_STATE);
        } else {
            markBlockAsCorrupt(c, storageInfo, node);
        }
        return;
    }
    if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
        addStoredBlockUnderConstruction(new StatefulBlockInfo(storedBlock, new Block(block), reportedState), storageInfo);
        return;
    }
    // but now okay, it might need to be updated.
    if (reportedState == ReplicaState.FINALIZED && (storedBlock.findStorageInfo(storageInfo) == -1 || corruptReplicas.isReplicaCorrupt(storedBlock, node))) {
        addStoredBlock(storedBlock, block, storageInfo, delHintNode, true);
    }
}

Example 4 with BlockUCState

use of org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState in project hadoop by apache.

the class FSNamesystem method internalReleaseLease.

/**
   * Move a file that is being written to be immutable.
   * @param src The filename
   * @param lease The lease for the client creating the file
   * @param recoveryLeaseHolder reassign lease to this holder if the last block
   *        needs recovery; keep current holder if null.
   * @throws AlreadyBeingCreatedException if file is waiting to achieve minimal
   *         replication;<br>
   *         RecoveryInProgressException if lease recovery is in progress.<br>
   *         IOException in case of an error.
   * @return true  if file has been successfully finalized and closed or 
   *         false if block recovery has been initiated. Since the lease owner
   *         has been changed and logged, caller should call logSync().
   */
boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, String recoveryLeaseHolder) throws IOException {
    LOG.info("Recovering " + lease + ", src=" + src);
    assert !isInSafeMode();
    assert hasWriteLock();
    final INodeFile pendingFile = iip.getLastINode().asFile();
    int nrBlocks = pendingFile.numBlocks();
    BlockInfo[] blocks = pendingFile.getBlocks();
    int nrCompleteBlocks;
    BlockInfo curBlock = null;
    for (nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
        curBlock = blocks[nrCompleteBlocks];
        if (!curBlock.isComplete())
            break;
        assert blockManager.hasMinStorage(curBlock) : "A COMPLETE block is not minimally replicated in " + src;
    }
    // then reap lease immediately and close the file.
    if (nrCompleteBlocks == nrBlocks) {
        finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
        NameNode.stateChangeLog.warn("BLOCK*" + " internalReleaseLease: All existing blocks are COMPLETE," + " lease removed, file " + src + " closed.");
        // closed!
        return true;
    }
    // If the penultimate block is not COMPLETE, then it must be COMMITTED.
    if (nrCompleteBlocks < nrBlocks - 2 || nrCompleteBlocks == nrBlocks - 2 && curBlock != null && curBlock.getBlockUCState() != BlockUCState.COMMITTED) {
        final String message = "DIR* NameSystem.internalReleaseLease: " + "attempt to release a create lock on " + src + " but file is already closed.";
        NameNode.stateChangeLog.warn(message);
        throw new IOException(message);
    }
    // The last block is not COMPLETE, and
    // that the penultimate block if exists is either COMPLETE or COMMITTED
    final BlockInfo lastBlock = pendingFile.getLastBlock();
    BlockUCState lastBlockState = lastBlock.getBlockUCState();
    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
    // If penultimate block doesn't exist then its minReplication is met
    boolean penultimateBlockMinStorage = penultimateBlock == null || blockManager.hasMinStorage(penultimateBlock);
    switch(lastBlockState) {
        case COMPLETE:
            assert false : "Already checked that the last block is incomplete";
            break;
        case COMMITTED:
            // Close file if committed blocks are minimally replicated
            if (penultimateBlockMinStorage && blockManager.hasMinStorage(lastBlock)) {
                finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
                NameNode.stateChangeLog.warn("BLOCK*" + " internalReleaseLease: Committed blocks are minimally" + " replicated, lease removed, file" + src + " closed.");
                // closed!
                return true;
            }
            // Cannot close file right now, since some blocks 
            // are not yet minimally replicated.
            // This may potentially cause infinite loop in lease recovery
            // if there are no valid replicas on data-nodes.
            String message = "DIR* NameSystem.internalReleaseLease: " + "Failed to release lease for file " + src + ". Committed blocks are waiting to be minimally replicated." + " Try again later.";
            NameNode.stateChangeLog.warn(message);
            throw new AlreadyBeingCreatedException(message);
        case UNDER_CONSTRUCTION:
        case UNDER_RECOVERY:
            BlockUnderConstructionFeature uc = lastBlock.getUnderConstructionFeature();
            // determine if last block was intended to be truncated
            Block recoveryBlock = uc.getTruncateBlock();
            boolean truncateRecovery = recoveryBlock != null;
            boolean copyOnTruncate = truncateRecovery && recoveryBlock.getBlockId() != lastBlock.getBlockId();
            assert !copyOnTruncate || recoveryBlock.getBlockId() < lastBlock.getBlockId() && recoveryBlock.getGenerationStamp() < lastBlock.getGenerationStamp() && recoveryBlock.getNumBytes() > lastBlock.getNumBytes() : "wrong recoveryBlock";
            // setup the last block locations from the blockManager if not known
            if (uc.getNumExpectedLocations() == 0) {
                uc.setExpectedLocations(lastBlock, blockManager.getStorages(lastBlock), lastBlock.getBlockType());
            }
            if (uc.getNumExpectedLocations() == 0 && lastBlock.getNumBytes() == 0) {
                // There is no datanode reported to this block.
                // may be client have crashed before writing data to pipeline.
                // This blocks doesn't need any recovery.
                // We can remove this block and close the file.
                pendingFile.removeLastBlock(lastBlock);
                finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false);
                NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " + "Removed empty last block and closed file " + src);
                return true;
            }
            // start recovery of the last block for this file
            long blockRecoveryId = nextGenerationStamp(blockManager.isLegacyBlock(lastBlock));
            lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
            if (copyOnTruncate) {
                lastBlock.setGenerationStamp(blockRecoveryId);
            } else if (truncateRecovery) {
                recoveryBlock.setGenerationStamp(blockRecoveryId);
            }
            uc.initializeBlockRecovery(lastBlock, blockRecoveryId);
            leaseManager.renewLease(lease);
            // Cannot close file right now, since the last block requires recovery.
            // This may potentially cause infinite loop in lease recovery
            // if there are no valid replicas on data-nodes.
            NameNode.stateChangeLog.warn("DIR* NameSystem.internalReleaseLease: " + "File " + src + " has not been closed." + " Lease recovery is in progress. " + "RecoveryId = " + blockRecoveryId + " for block " + lastBlock);
            break;
    }
    return false;
}

Example 5 with BlockUCState

use of org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState in project hadoop by apache.

the class INodeFile method checkBlockComplete.

/**
   * Check if the i-th block is COMPLETE;
   * when the i-th block is the last block, it may be allowed to be COMMITTED.
   *
   * @return null if the block passes the check;
   *              otherwise, return an error message.
   */
static String checkBlockComplete(BlockInfo[] blocks, int i, int numCommittedAllowed, short minReplication) {
    final BlockInfo b = blocks[i];
    final BlockUCState state = b.getBlockUCState();
    if (state == BlockUCState.COMPLETE) {
        return null;
    }
    if (b.isStriped() || i < blocks.length - numCommittedAllowed) {
        return b + " is " + state + " but not COMPLETE";
    }
    if (state != BlockUCState.COMMITTED) {
        return b + " is " + state + " but neither COMPLETE nor COMMITTED";
    }
    final int numExpectedLocations = b.getUnderConstructionFeature().getNumExpectedLocations();
    if (numExpectedLocations <= minReplication) {
        return b + " is " + state + " but numExpectedLocations = " + numExpectedLocations + " <= minReplication = " + minReplication;
    }
    return null;
}