Examples with BlockStoragePolicy org.apache.hadoop.hdfs.protocol.BlockStoragePolicy used on opensource projects

Example 6 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class FSDirAppendOp method computeQuotaDeltaForUCBlock.

/** Compute quota change for converting a complete block to a UC block. */
private static QuotaCounts computeQuotaDeltaForUCBlock(FSNamesystem fsn, INodeFile file) {
    final QuotaCounts delta = new QuotaCounts.Builder().build();
    final BlockInfo lastBlock = file.getLastBlock();
    if (lastBlock != null) {
        final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes();
        final short repl = lastBlock.getReplication();
        delta.addStorageSpace(diff * repl);
        final BlockStoragePolicy policy = fsn.getFSDirectory().getBlockStoragePolicySuite().getPolicy(file.getStoragePolicyID());
        List<StorageType> types = policy.chooseStorageTypes(repl);
        for (StorageType t : types) {
            if (t.supportTypeQuota()) {
                delta.addTypeSpace(t, diff);
            }
        }
    }
    return delta;
}

Example 7 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class FSDirAppendOp method appendFile.

/**
   * Append to an existing file.
   * <p>
   *
   * The method returns the last block of the file if this is a partial block,
   * which can still be used for writing more data. The client uses the
   * returned block locations to form the data pipeline for this block.<br>
   * The {@link LocatedBlock} will be null if the last block is full.
   * The client then allocates a new block with the next call using
   * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#addBlock}.
   * <p>
   *
   * For description of parameters and exceptions thrown see
   * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#append}
   *
   * @param fsn namespace
   * @param srcArg path name
   * @param pc permission checker to check fs permission
   * @param holder client name
   * @param clientMachine client machine info
   * @param newBlock if the data is appended to a new block
   * @param logRetryCache whether to record RPC ids in editlog for retry cache
   *                      rebuilding
   * @return the last block with status
   */
static LastBlockWithStatus appendFile(final FSNamesystem fsn, final String srcArg, final FSPermissionChecker pc, final String holder, final String clientMachine, final boolean newBlock, final boolean logRetryCache) throws IOException {
    assert fsn.hasWriteLock();
    final LocatedBlock lb;
    final FSDirectory fsd = fsn.getFSDirectory();
    final INodesInPath iip;
    fsd.writeLock();
    try {
        iip = fsd.resolvePath(pc, srcArg, DirOp.WRITE);
        // Verify that the destination does not exist as a directory already
        final INode inode = iip.getLastINode();
        final String path = iip.getPath();
        if (inode != null && inode.isDirectory()) {
            throw new FileAlreadyExistsException("Cannot append to directory " + path + "; already exists as a directory.");
        }
        if (fsd.isPermissionEnabled()) {
            fsd.checkPathAccess(pc, iip, FsAction.WRITE);
        }
        if (inode == null) {
            throw new FileNotFoundException("Failed to append to non-existent file " + path + " for client " + clientMachine);
        }
        final INodeFile file = INodeFile.valueOf(inode, path, true);
        // not support appending file with striped blocks
        if (file.isStriped()) {
            throw new UnsupportedOperationException("Cannot append to files with striped block " + path);
        }
        BlockManager blockManager = fsd.getBlockManager();
        final BlockStoragePolicy lpPolicy = blockManager.getStoragePolicy("LAZY_PERSIST");
        if (lpPolicy != null && lpPolicy.getId() == file.getStoragePolicyID()) {
            throw new UnsupportedOperationException("Cannot append to lazy persist file " + path);
        }
        // Opening an existing file for append - may need to recover lease.
        fsn.recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE, iip, path, holder, clientMachine, false);
        final BlockInfo lastBlock = file.getLastBlock();
        // Check that the block has at least minimum replication.
        if (lastBlock != null) {
            if (lastBlock.getBlockUCState() == BlockUCState.COMMITTED) {
                throw new RetriableException(new NotReplicatedYetException("append: lastBlock=" + lastBlock + " of src=" + path + " is COMMITTED but not yet COMPLETE."));
            } else if (lastBlock.isComplete() && !blockManager.isSufficientlyReplicated(lastBlock)) {
                throw new IOException("append: lastBlock=" + lastBlock + " of src=" + path + " is not sufficiently replicated yet.");
            }
        }
        lb = prepareFileForAppend(fsn, iip, holder, clientMachine, newBlock, true, logRetryCache);
    } catch (IOException ie) {
        NameNode.stateChangeLog.warn("DIR* NameSystem.append: " + ie.getMessage());
        throw ie;
    } finally {
        fsd.writeUnlock();
    }
    HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
    if (lb != null) {
        NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: file {} for {} at {} block {} block" + " size {}", srcArg, holder, clientMachine, lb.getBlock(), lb.getBlock().getNumBytes());
    }
    return new LastBlockWithStatus(lb, stat);
}

Example 8 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class FSDirAttrOp method unprotectedSetStoragePolicy.

static void unprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm, INodesInPath iip, final byte policyId) throws IOException {
    assert fsd.hasWriteLock();
    final INode inode = iip.getLastINode();
    if (inode == null) {
        throw new FileNotFoundException("File/Directory does not exist: " + iip.getPath());
    }
    final int snapshotId = iip.getLatestSnapshotId();
    if (inode.isFile()) {
        if (policyId != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
            BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
            if (newPolicy.isCopyOnCreateFile()) {
                throw new HadoopIllegalArgumentException("Policy " + newPolicy + " cannot be set after file creation.");
            }
        }
        BlockStoragePolicy currentPolicy = bm.getStoragePolicy(inode.getLocalStoragePolicyID());
        if (currentPolicy != null && currentPolicy.isCopyOnCreateFile()) {
            throw new HadoopIllegalArgumentException("Existing policy " + currentPolicy.getName() + " cannot be changed after file creation.");
        }
        inode.asFile().setStoragePolicyID(policyId, snapshotId);
    } else if (inode.isDirectory()) {
        setDirStoragePolicy(fsd, iip, policyId);
    } else {
        throw new FileNotFoundException(iip.getPath() + " is not a file or directory");
    }
}

Example 9 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class FSDirConcatOp method computeQuotaDeltas.

private static QuotaCounts computeQuotaDeltas(FSDirectory fsd, INodeFile target, INodeFile[] srcList) {
    QuotaCounts deltas = new QuotaCounts.Builder().build();
    final short targetRepl = target.getPreferredBlockReplication();
    for (INodeFile src : srcList) {
        short srcRepl = src.getFileReplication();
        long fileSize = src.computeFileSize();
        if (targetRepl != srcRepl) {
            deltas.addStorageSpace(fileSize * (targetRepl - srcRepl));
            BlockStoragePolicy bsp = fsd.getBlockStoragePolicySuite().getPolicy(src.getStoragePolicyID());
            if (bsp != null) {
                List<StorageType> srcTypeChosen = bsp.chooseStorageTypes(srcRepl);
                for (StorageType t : srcTypeChosen) {
                    if (t.supportTypeQuota()) {
                        deltas.addTypeSpace(t, -fileSize);
                    }
                }
                List<StorageType> targetTypeChosen = bsp.chooseStorageTypes(targetRepl);
                for (StorageType t : targetTypeChosen) {
                    if (t.supportTypeQuota()) {
                        deltas.addTypeSpace(t, fileSize);
                    }
                }
            }
        }
    }
    return deltas;
}

Example 10 with BlockStoragePolicy

use of org.apache.hadoop.hdfs.protocol.BlockStoragePolicy in project hadoop by apache.

the class FSDirTruncateOp method verifyQuotaForTruncate.

private static void verifyQuotaForTruncate(FSNamesystem fsn, INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta) throws QuotaExceededException {
    FSDirectory fsd = fsn.getFSDirectory();
    if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
        // Do not check quota if edit log is still being processed
        return;
    }
    final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite().getPolicy(file.getStoragePolicyID());
    file.computeQuotaDeltaForTruncate(newLength, policy, delta);
    fsd.readLock();
    try {
        FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null);
    } finally {
        fsd.readUnlock();
    }
}