use of org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo in project hadoop by apache.
the class FSEditLogLoader method applyEditLogOp.
@SuppressWarnings("deprecation")
private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, StartupOption startOpt, int logVersion, long lastInodeId) throws IOException {
long inodeId = HdfsConstants.GRANDFATHER_INODE_ID;
if (LOG.isTraceEnabled()) {
LOG.trace("replaying edit log: " + op);
}
final boolean toAddRetryCache = fsNamesys.hasRetryCache() && op.hasRpcIds();
switch(op.opCode) {
case OP_ADD:
{
AddCloseOp addCloseOp = (AddCloseOp) op;
final String path = renameReservedPathsOnUpgrade(addCloseOp.path, logVersion);
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine);
}
// There are 3 cases here:
// 1. OP_ADD to create a new file
// 2. OP_ADD to update file blocks
// 3. OP_ADD to open file for append (old append)
// See if the file already exists (persistBlocks call)
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.WRITE);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path, true);
if (oldFile != null && addCloseOp.overwrite) {
// This is OP_ADD with overwrite
FSDirDeleteOp.deleteForEditLog(fsDir, iip, addCloseOp.mtime);
iip = INodesInPath.replace(iip, iip.length() - 1, null);
oldFile = null;
}
INodeFile newFile = oldFile;
if (oldFile == null) {
// this is OP_ADD on a new file (case 1)
// versions > 0 support per file replication
// get name and replication
final short replication = fsNamesys.getBlockManager().adjustReplication(addCloseOp.replication);
assert addCloseOp.blocks.length == 0;
// add to the file tree
inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId);
newFile = FSDirWriteFileOp.addFileForEditLog(fsDir, inodeId, iip.getExistingINodes(), iip.getLastLocalName(), addCloseOp.permissions, addCloseOp.aclEntries, addCloseOp.xAttrs, replication, addCloseOp.mtime, addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, addCloseOp.clientMachine, addCloseOp.storagePolicyId);
assert newFile != null;
iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, newFile.getId());
// add the op into retry cache if necessary
if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip);
fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, addCloseOp.rpcCallId, stat);
}
} else {
// This is OP_ADD on an existing file (old append)
if (!oldFile.isUnderConstruction()) {
// This is case 3: a call to append() on an already-closed file.
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug("Reopening an already-closed file " + "for append");
}
LocatedBlock lb = FSDirAppendOp.prepareFileForAppend(fsNamesys, iip, addCloseOp.clientName, addCloseOp.clientMachine, false, false, false);
// add the op into retry cache if necessary
if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip);
fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, addCloseOp.rpcCallId, new LastBlockWithStatus(lb, stat));
}
}
}
// Fall-through for case 2.
// Regardless of whether it's a new file or an updated file,
// update the block list.
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy);
break;
}
case OP_CLOSE:
{
AddCloseOp addCloseOp = (AddCloseOp) op;
final String path = renameReservedPathsOnUpgrade(addCloseOp.path, logVersion);
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine);
}
final INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
final INodeFile file = INodeFile.valueOf(iip.getLastINode(), path);
// Update the salient file attributes.
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy);
// Now close the file
if (!file.isUnderConstruction() && logVersion <= LayoutVersion.BUGFIX_HDFS_2991_VERSION) {
// should be fixed, so we should treat it as an error.
throw new IOException("File is not under construction: " + path);
}
// but OP_CLOSE doesn't serialize the holder. So, remove the inode.
if (file.isUnderConstruction()) {
fsNamesys.getLeaseManager().removeLease(file.getId());
file.toCompleteFile(file.getModificationTime(), 0, fsNamesys.getBlockManager().getMinReplication());
}
break;
}
case OP_APPEND:
{
AppendOp appendOp = (AppendOp) op;
final String path = renameReservedPathsOnUpgrade(appendOp.path, logVersion);
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(op.opCode + ": " + path + " clientName " + appendOp.clientName + " clientMachine " + appendOp.clientMachine + " newBlock " + appendOp.newBlock);
}
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.WRITE);
INodeFile file = INodeFile.valueOf(iip.getLastINode(), path);
if (!file.isUnderConstruction()) {
LocatedBlock lb = FSDirAppendOp.prepareFileForAppend(fsNamesys, iip, appendOp.clientName, appendOp.clientMachine, appendOp.newBlock, false, false);
// add the op into retry cache if necessary
if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip);
fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId, appendOp.rpcCallId, new LastBlockWithStatus(lb, stat));
}
}
break;
}
case OP_UPDATE_BLOCKS:
{
UpdateBlocksOp updateOp = (UpdateBlocksOp) op;
final String path = renameReservedPathsOnUpgrade(updateOp.path, logVersion);
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + updateOp.blocks.length);
}
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsDir.getFSNamesystem(), iip);
updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
}
break;
}
case OP_ADD_BLOCK:
{
AddBlockOp addBlockOp = (AddBlockOp) op;
String path = renameReservedPathsOnUpgrade(addBlockOp.getPath(), logVersion);
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(op.opCode + ": " + path + " new block id : " + addBlockOp.getLastBlock().getBlockId());
}
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsDir.getFSNamesystem(), iip);
addNewBlock(addBlockOp, oldFile, ecPolicy);
break;
}
case OP_SET_REPLICATION:
{
SetReplicationOp setReplicationOp = (SetReplicationOp) op;
String src = renameReservedPathsOnUpgrade(setReplicationOp.path, logVersion);
INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
short replication = fsNamesys.getBlockManager().adjustReplication(setReplicationOp.replication);
FSDirAttrOp.unprotectedSetReplication(fsDir, iip, replication);
break;
}
case OP_CONCAT_DELETE:
{
ConcatDeleteOp concatDeleteOp = (ConcatDeleteOp) op;
String trg = renameReservedPathsOnUpgrade(concatDeleteOp.trg, logVersion);
String[] srcs = new String[concatDeleteOp.srcs.length];
for (int i = 0; i < srcs.length; i++) {
srcs[i] = renameReservedPathsOnUpgrade(concatDeleteOp.srcs[i], logVersion);
}
INodesInPath targetIIP = fsDir.getINodesInPath(trg, DirOp.WRITE);
INodeFile[] srcFiles = new INodeFile[srcs.length];
for (int i = 0; i < srcs.length; i++) {
INodesInPath srcIIP = fsDir.getINodesInPath(srcs[i], DirOp.WRITE);
srcFiles[i] = srcIIP.getLastINode().asFile();
}
FSDirConcatOp.unprotectedConcat(fsDir, targetIIP, srcFiles, concatDeleteOp.timestamp);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(concatDeleteOp.rpcClientId, concatDeleteOp.rpcCallId);
}
break;
}
case OP_RENAME_OLD:
{
RenameOldOp renameOp = (RenameOldOp) op;
final String src = renameReservedPathsOnUpgrade(renameOp.src, logVersion);
final String dst = renameReservedPathsOnUpgrade(renameOp.dst, logVersion);
FSDirRenameOp.renameForEditLog(fsDir, src, dst, renameOp.timestamp);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(renameOp.rpcClientId, renameOp.rpcCallId);
}
break;
}
case OP_DELETE:
{
DeleteOp deleteOp = (DeleteOp) op;
final String src = renameReservedPathsOnUpgrade(deleteOp.path, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE_LINK);
FSDirDeleteOp.deleteForEditLog(fsDir, iip, deleteOp.timestamp);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(deleteOp.rpcClientId, deleteOp.rpcCallId);
}
break;
}
case OP_MKDIR:
{
MkdirOp mkdirOp = (MkdirOp) op;
inodeId = getAndUpdateLastInodeId(mkdirOp.inodeId, logVersion, lastInodeId);
FSDirMkdirOp.mkdirForEditLog(fsDir, inodeId, renameReservedPathsOnUpgrade(mkdirOp.path, logVersion), mkdirOp.permissions, mkdirOp.aclEntries, mkdirOp.timestamp);
break;
}
case OP_SET_GENSTAMP_V1:
{
SetGenstampV1Op setGenstampV1Op = (SetGenstampV1Op) op;
blockManager.getBlockIdManager().setLegacyGenerationStamp(setGenstampV1Op.genStampV1);
break;
}
case OP_SET_PERMISSIONS:
{
SetPermissionsOp setPermissionsOp = (SetPermissionsOp) op;
final String src = renameReservedPathsOnUpgrade(setPermissionsOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetPermission(fsDir, iip, setPermissionsOp.permissions);
break;
}
case OP_SET_OWNER:
{
SetOwnerOp setOwnerOp = (SetOwnerOp) op;
final String src = renameReservedPathsOnUpgrade(setOwnerOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetOwner(fsDir, iip, setOwnerOp.username, setOwnerOp.groupname);
break;
}
case OP_SET_NS_QUOTA:
{
SetNSQuotaOp setNSQuotaOp = (SetNSQuotaOp) op;
final String src = renameReservedPathsOnUpgrade(setNSQuotaOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetQuota(fsDir, iip, setNSQuotaOp.nsQuota, HdfsConstants.QUOTA_DONT_SET, null);
break;
}
case OP_CLEAR_NS_QUOTA:
{
ClearNSQuotaOp clearNSQuotaOp = (ClearNSQuotaOp) op;
final String src = renameReservedPathsOnUpgrade(clearNSQuotaOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetQuota(fsDir, iip, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET, null);
break;
}
case OP_SET_QUOTA:
{
SetQuotaOp setQuotaOp = (SetQuotaOp) op;
final String src = renameReservedPathsOnUpgrade(setQuotaOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetQuota(fsDir, iip, setQuotaOp.nsQuota, setQuotaOp.dsQuota, null);
break;
}
case OP_SET_QUOTA_BY_STORAGETYPE:
{
FSEditLogOp.SetQuotaByStorageTypeOp setQuotaByStorageTypeOp = (FSEditLogOp.SetQuotaByStorageTypeOp) op;
final String src = renameReservedPathsOnUpgrade(setQuotaByStorageTypeOp.src, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetQuota(fsDir, iip, HdfsConstants.QUOTA_DONT_SET, setQuotaByStorageTypeOp.dsQuota, setQuotaByStorageTypeOp.type);
break;
}
case OP_TIMES:
{
TimesOp timesOp = (TimesOp) op;
final String src = renameReservedPathsOnUpgrade(timesOp.path, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(src, DirOp.WRITE);
FSDirAttrOp.unprotectedSetTimes(fsDir, iip, timesOp.mtime, timesOp.atime, true);
break;
}
case OP_SYMLINK:
{
if (!FileSystem.areSymlinksEnabled()) {
throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS");
}
SymlinkOp symlinkOp = (SymlinkOp) op;
inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion, lastInodeId);
final String path = renameReservedPathsOnUpgrade(symlinkOp.path, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(path, DirOp.WRITE_LINK);
FSDirSymlinkOp.unprotectedAddSymlink(fsDir, iip.getExistingINodes(), iip.getLastLocalName(), inodeId, symlinkOp.value, symlinkOp.mtime, symlinkOp.atime, symlinkOp.permissionStatus);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(symlinkOp.rpcClientId, symlinkOp.rpcCallId);
}
break;
}
case OP_RENAME:
{
RenameOp renameOp = (RenameOp) op;
FSDirRenameOp.renameForEditLog(fsDir, renameReservedPathsOnUpgrade(renameOp.src, logVersion), renameReservedPathsOnUpgrade(renameOp.dst, logVersion), renameOp.timestamp, renameOp.options);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(renameOp.rpcClientId, renameOp.rpcCallId);
}
break;
}
case OP_GET_DELEGATION_TOKEN:
{
GetDelegationTokenOp getDelegationTokenOp = (GetDelegationTokenOp) op;
fsNamesys.getDelegationTokenSecretManager().addPersistedDelegationToken(getDelegationTokenOp.token, getDelegationTokenOp.expiryTime);
break;
}
case OP_RENEW_DELEGATION_TOKEN:
{
RenewDelegationTokenOp renewDelegationTokenOp = (RenewDelegationTokenOp) op;
fsNamesys.getDelegationTokenSecretManager().updatePersistedTokenRenewal(renewDelegationTokenOp.token, renewDelegationTokenOp.expiryTime);
break;
}
case OP_CANCEL_DELEGATION_TOKEN:
{
CancelDelegationTokenOp cancelDelegationTokenOp = (CancelDelegationTokenOp) op;
fsNamesys.getDelegationTokenSecretManager().updatePersistedTokenCancellation(cancelDelegationTokenOp.token);
break;
}
case OP_UPDATE_MASTER_KEY:
{
UpdateMasterKeyOp updateMasterKeyOp = (UpdateMasterKeyOp) op;
fsNamesys.getDelegationTokenSecretManager().updatePersistedMasterKey(updateMasterKeyOp.key);
break;
}
case OP_REASSIGN_LEASE:
{
ReassignLeaseOp reassignLeaseOp = (ReassignLeaseOp) op;
Lease lease = fsNamesys.leaseManager.getLease(reassignLeaseOp.leaseHolder);
final String path = renameReservedPathsOnUpgrade(reassignLeaseOp.path, logVersion);
INodeFile pendingFile = fsDir.getINode(path, DirOp.READ).asFile();
Preconditions.checkState(pendingFile.isUnderConstruction());
fsNamesys.reassignLeaseInternal(lease, reassignLeaseOp.newHolder, pendingFile);
break;
}
case OP_START_LOG_SEGMENT:
case OP_END_LOG_SEGMENT:
{
// no data in here currently.
break;
}
case OP_CREATE_SNAPSHOT:
{
CreateSnapshotOp createSnapshotOp = (CreateSnapshotOp) op;
final String snapshotRoot = renameReservedPathsOnUpgrade(createSnapshotOp.snapshotRoot, logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
String path = fsNamesys.getSnapshotManager().createSnapshot(iip, snapshotRoot, createSnapshotOp.snapshotName);
if (toAddRetryCache) {
fsNamesys.addCacheEntryWithPayload(createSnapshotOp.rpcClientId, createSnapshotOp.rpcCallId, path);
}
break;
}
case OP_DELETE_SNAPSHOT:
{
DeleteSnapshotOp deleteSnapshotOp = (DeleteSnapshotOp) op;
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
List<INode> removedINodes = new ChunkedArrayList<INode>();
final String snapshotRoot = renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot, logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
fsNamesys.getSnapshotManager().deleteSnapshot(iip, deleteSnapshotOp.snapshotName, new INode.ReclaimContext(fsNamesys.dir.getBlockStoragePolicySuite(), collectedBlocks, removedINodes, null));
fsNamesys.getBlockManager().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
collectedBlocks.clear();
fsNamesys.dir.removeFromInodeMap(removedINodes);
removedINodes.clear();
if (toAddRetryCache) {
fsNamesys.addCacheEntry(deleteSnapshotOp.rpcClientId, deleteSnapshotOp.rpcCallId);
}
break;
}
case OP_RENAME_SNAPSHOT:
{
RenameSnapshotOp renameSnapshotOp = (RenameSnapshotOp) op;
final String snapshotRoot = renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot, logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
fsNamesys.getSnapshotManager().renameSnapshot(iip, snapshotRoot, renameSnapshotOp.snapshotOldName, renameSnapshotOp.snapshotNewName);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(renameSnapshotOp.rpcClientId, renameSnapshotOp.rpcCallId);
}
break;
}
case OP_ALLOW_SNAPSHOT:
{
AllowSnapshotOp allowSnapshotOp = (AllowSnapshotOp) op;
final String snapshotRoot = renameReservedPathsOnUpgrade(allowSnapshotOp.snapshotRoot, logVersion);
fsNamesys.getSnapshotManager().setSnapshottable(snapshotRoot, false);
break;
}
case OP_DISALLOW_SNAPSHOT:
{
DisallowSnapshotOp disallowSnapshotOp = (DisallowSnapshotOp) op;
final String snapshotRoot = renameReservedPathsOnUpgrade(disallowSnapshotOp.snapshotRoot, logVersion);
fsNamesys.getSnapshotManager().resetSnapshottable(snapshotRoot);
break;
}
case OP_SET_GENSTAMP_V2:
{
SetGenstampV2Op setGenstampV2Op = (SetGenstampV2Op) op;
blockManager.getBlockIdManager().setGenerationStamp(setGenstampV2Op.genStampV2);
break;
}
case OP_ALLOCATE_BLOCK_ID:
{
AllocateBlockIdOp allocateBlockIdOp = (AllocateBlockIdOp) op;
if (BlockIdManager.isStripedBlockID(allocateBlockIdOp.blockId)) {
// ALLOCATE_BLOCK_ID is added for sequential block id, thus if the id
// is negative, it must belong to striped blocks
blockManager.getBlockIdManager().setLastAllocatedStripedBlockId(allocateBlockIdOp.blockId);
} else {
blockManager.getBlockIdManager().setLastAllocatedContiguousBlockId(allocateBlockIdOp.blockId);
}
break;
}
case OP_ROLLING_UPGRADE_START:
{
if (startOpt == StartupOption.ROLLINGUPGRADE) {
final RollingUpgradeStartupOption rollingUpgradeOpt = startOpt.getRollingUpgradeStartupOption();
if (rollingUpgradeOpt == RollingUpgradeStartupOption.ROLLBACK) {
throw new RollingUpgradeOp.RollbackException();
}
}
// start rolling upgrade
final long startTime = ((RollingUpgradeOp) op).getTime();
fsNamesys.startRollingUpgradeInternal(startTime);
fsNamesys.triggerRollbackCheckpoint();
break;
}
case OP_ROLLING_UPGRADE_FINALIZE:
{
final long finalizeTime = ((RollingUpgradeOp) op).getTime();
if (fsNamesys.isRollingUpgrade()) {
// Only do it when NN is actually doing rolling upgrade.
// We can get FINALIZE without corresponding START, if NN is restarted
// before this op is consumed and a new checkpoint is created.
fsNamesys.finalizeRollingUpgradeInternal(finalizeTime);
}
fsNamesys.getFSImage().updateStorageVersion();
fsNamesys.getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE);
break;
}
case OP_ADD_CACHE_DIRECTIVE:
{
AddCacheDirectiveInfoOp addOp = (AddCacheDirectiveInfoOp) op;
CacheDirectiveInfo result = fsNamesys.getCacheManager().addDirectiveFromEditLog(addOp.directive);
if (toAddRetryCache) {
Long id = result.getId();
fsNamesys.addCacheEntryWithPayload(op.rpcClientId, op.rpcCallId, id);
}
break;
}
case OP_MODIFY_CACHE_DIRECTIVE:
{
ModifyCacheDirectiveInfoOp modifyOp = (ModifyCacheDirectiveInfoOp) op;
fsNamesys.getCacheManager().modifyDirectiveFromEditLog(modifyOp.directive);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
break;
}
case OP_REMOVE_CACHE_DIRECTIVE:
{
RemoveCacheDirectiveInfoOp removeOp = (RemoveCacheDirectiveInfoOp) op;
fsNamesys.getCacheManager().removeDirective(removeOp.id, null);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
break;
}
case OP_ADD_CACHE_POOL:
{
AddCachePoolOp addOp = (AddCachePoolOp) op;
fsNamesys.getCacheManager().addCachePool(addOp.info);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
break;
}
case OP_MODIFY_CACHE_POOL:
{
ModifyCachePoolOp modifyOp = (ModifyCachePoolOp) op;
fsNamesys.getCacheManager().modifyCachePool(modifyOp.info);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
break;
}
case OP_REMOVE_CACHE_POOL:
{
RemoveCachePoolOp removeOp = (RemoveCachePoolOp) op;
fsNamesys.getCacheManager().removeCachePool(removeOp.poolName);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId);
}
break;
}
case OP_SET_ACL:
{
SetAclOp setAclOp = (SetAclOp) op;
INodesInPath iip = fsDir.getINodesInPath(setAclOp.src, DirOp.WRITE);
FSDirAclOp.unprotectedSetAcl(fsDir, iip, setAclOp.aclEntries, true);
break;
}
case OP_SET_XATTR:
{
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
INodesInPath iip = fsDir.getINodesInPath(setXAttrOp.src, DirOp.WRITE);
FSDirXAttrOp.unprotectedSetXAttrs(fsDir, iip, setXAttrOp.xAttrs, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
}
break;
}
case OP_REMOVE_XATTR:
{
RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
FSDirXAttrOp.unprotectedRemoveXAttrs(fsDir, removeXAttrOp.src, removeXAttrOp.xAttrs);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(removeXAttrOp.rpcClientId, removeXAttrOp.rpcCallId);
}
break;
}
case OP_TRUNCATE:
{
TruncateOp truncateOp = (TruncateOp) op;
INodesInPath iip = fsDir.getINodesInPath(truncateOp.src, DirOp.WRITE);
FSDirTruncateOp.unprotectedTruncate(fsNamesys, iip, truncateOp.clientName, truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp, truncateOp.truncateBlock);
break;
}
case OP_SET_STORAGE_POLICY:
{
SetStoragePolicyOp setStoragePolicyOp = (SetStoragePolicyOp) op;
final String path = renameReservedPathsOnUpgrade(setStoragePolicyOp.path, logVersion);
final INodesInPath iip = fsDir.getINodesInPath(path, DirOp.WRITE);
FSDirAttrOp.unprotectedSetStoragePolicy(fsDir, fsNamesys.getBlockManager(), iip, setStoragePolicyOp.policyId);
break;
}
default:
throw new IOException("Invalid operation read " + op.opCode);
}
return inodeId;
}
use of org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo in project hadoop by apache.
the class CacheManager method listCacheDirectives.
public BatchedListEntries<CacheDirectiveEntry> listCacheDirectives(long prevId, CacheDirectiveInfo filter, FSPermissionChecker pc) throws IOException {
assert namesystem.hasReadLock();
final int NUM_PRE_ALLOCATED_ENTRIES = 16;
String filterPath = null;
if (filter.getPath() != null) {
filterPath = validatePath(filter);
}
if (filter.getReplication() != null) {
throw new InvalidRequestException("Filtering by replication is unsupported.");
}
// Querying for a single ID
final Long id = filter.getId();
if (id != null) {
if (!directivesById.containsKey(id)) {
throw new InvalidRequestException("Did not find requested id " + id);
}
// Since we use a tailMap on directivesById, setting prev to id-1 gets
// us the directive with the id (if present)
prevId = id - 1;
}
ArrayList<CacheDirectiveEntry> replies = new ArrayList<CacheDirectiveEntry>(NUM_PRE_ALLOCATED_ENTRIES);
int numReplies = 0;
SortedMap<Long, CacheDirective> tailMap = directivesById.tailMap(prevId + 1);
for (Entry<Long, CacheDirective> cur : tailMap.entrySet()) {
if (numReplies >= maxListCacheDirectivesNumResponses) {
return new BatchedListEntries<CacheDirectiveEntry>(replies, true);
}
CacheDirective curDirective = cur.getValue();
CacheDirectiveInfo info = cur.getValue().toInfo();
// item and should break out.
if (id != null && !(info.getId().equals(id))) {
break;
}
if (filter.getPool() != null && !info.getPool().equals(filter.getPool())) {
continue;
}
if (filterPath != null && !info.getPath().toUri().getPath().equals(filterPath)) {
continue;
}
boolean hasPermission = true;
if (pc != null) {
try {
pc.checkPermission(curDirective.getPool(), FsAction.READ);
} catch (AccessControlException e) {
hasPermission = false;
}
}
if (hasPermission) {
replies.add(new CacheDirectiveEntry(info, cur.getValue().toStats()));
numReplies++;
}
}
return new BatchedListEntries<CacheDirectiveEntry>(replies, false);
}
use of org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo in project hadoop by apache.
the class CacheManager method modifyDirective.
public void modifyDirective(CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet<CacheFlag> flags) throws IOException {
assert namesystem.hasWriteLock();
String idString = (info.getId() == null) ? "(null)" : info.getId().toString();
try {
// Check for invalid IDs.
Long id = info.getId();
if (id == null) {
throw new InvalidRequestException("Must supply an ID.");
}
CacheDirective prevEntry = getById(id);
checkWritePermission(pc, prevEntry.getPool());
// Fill in defaults
CacheDirectiveInfo infoWithDefaults = createFromInfoAndDefaults(info, prevEntry);
CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(infoWithDefaults);
// Do validation
validatePath(infoWithDefaults);
validateReplication(infoWithDefaults, (short) -1);
// Need to test the pool being set here to avoid rejecting a modify for a
// directive that's already been forced into a pool
CachePool srcPool = prevEntry.getPool();
CachePool destPool = getCachePool(validatePoolName(infoWithDefaults));
if (!srcPool.getPoolName().equals(destPool.getPoolName())) {
checkWritePermission(pc, destPool);
if (!flags.contains(CacheFlag.FORCE)) {
checkLimit(destPool, infoWithDefaults.getPath().toUri().getPath(), infoWithDefaults.getReplication());
}
}
// Verify the expiration against the destination pool
validateExpiryTime(infoWithDefaults, destPool.getMaxRelativeExpiryMs());
// Indicate changes to the CRM
setNeedsRescan();
// Validation passed
removeInternal(prevEntry);
addInternal(new CacheDirective(builder.build()), destPool);
} catch (IOException e) {
LOG.warn("modifyDirective of " + idString + " failed: ", e);
throw e;
}
LOG.info("modifyDirective of {} successfully applied {}.", idString, info);
}
use of org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo in project hadoop by apache.
the class CacheManager method modifyDirectiveFromEditLog.
/**
* Modifies a directive, skipping most error checking. This is for careful
* internal use only. modifyDirective can be non-deterministic since its error
* checking depends on current system time, which poses a problem for edit log
* replay.
*/
void modifyDirectiveFromEditLog(CacheDirectiveInfo info) throws InvalidRequestException {
// Check for invalid IDs.
Long id = info.getId();
if (id == null) {
throw new InvalidRequestException("Must supply an ID.");
}
CacheDirective prevEntry = getById(id);
CacheDirectiveInfo newInfo = createFromInfoAndDefaults(info, prevEntry);
removeInternal(prevEntry);
addInternal(new CacheDirective(newInfo), getCachePool(newInfo.getPool()));
}
use of org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo in project hadoop by apache.
the class TestCacheDirectives method testCacheManagerRestart.
@Test(timeout = 60000)
public void testCacheManagerRestart() throws Exception {
SecondaryNameNode secondary = null;
try {
// Start a secondary namenode
conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, "0.0.0.0:0");
secondary = new SecondaryNameNode(conf);
// Create and validate a pool
final String pool = "poolparty";
String groupName = "partygroup";
FsPermission mode = new FsPermission((short) 0777);
long limit = 747;
dfs.addCachePool(new CachePoolInfo(pool).setGroupName(groupName).setMode(mode).setLimit(limit));
RemoteIterator<CachePoolEntry> pit = dfs.listCachePools();
assertTrue("No cache pools found", pit.hasNext());
CachePoolInfo info = pit.next().getInfo();
assertEquals(pool, info.getPoolName());
assertEquals(groupName, info.getGroupName());
assertEquals(mode, info.getMode());
assertEquals(limit, (long) info.getLimit());
assertFalse("Unexpected # of cache pools found", pit.hasNext());
// Create some cache entries
int numEntries = 10;
String entryPrefix = "/party-";
long prevId = -1;
final Date expiry = new Date();
for (int i = 0; i < numEntries; i++) {
prevId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path(entryPrefix + i)).setPool(pool).setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(expiry.getTime())).build());
}
RemoteIterator<CacheDirectiveEntry> dit = dfs.listCacheDirectives(null);
for (int i = 0; i < numEntries; i++) {
assertTrue("Unexpected # of cache entries: " + i, dit.hasNext());
CacheDirectiveInfo cd = dit.next().getInfo();
assertEquals(i + 1, cd.getId().longValue());
assertEquals(entryPrefix + i, cd.getPath().toUri().getPath());
assertEquals(pool, cd.getPool());
}
assertFalse("Unexpected # of cache directives found", dit.hasNext());
// Checkpoint once to set some cache pools and directives on 2NN side
secondary.doCheckpoint();
// Add some more CacheManager state
final String imagePool = "imagePool";
dfs.addCachePool(new CachePoolInfo(imagePool));
prevId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path("/image")).setPool(imagePool).build());
// Save a new image to force a fresh fsimage download
dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
dfs.saveNamespace();
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
// Checkpoint again forcing a reload of FSN state
boolean fetchImage = secondary.doCheckpoint();
assertTrue("Secondary should have fetched a new fsimage from NameNode", fetchImage);
// Remove temp pool and directive
dfs.removeCachePool(imagePool);
// Restart namenode
cluster.restartNameNode();
// Check that state came back up
pit = dfs.listCachePools();
assertTrue("No cache pools found", pit.hasNext());
info = pit.next().getInfo();
assertEquals(pool, info.getPoolName());
assertEquals(pool, info.getPoolName());
assertEquals(groupName, info.getGroupName());
assertEquals(mode, info.getMode());
assertEquals(limit, (long) info.getLimit());
assertFalse("Unexpected # of cache pools found", pit.hasNext());
dit = dfs.listCacheDirectives(null);
for (int i = 0; i < numEntries; i++) {
assertTrue("Unexpected # of cache entries: " + i, dit.hasNext());
CacheDirectiveInfo cd = dit.next().getInfo();
assertEquals(i + 1, cd.getId().longValue());
assertEquals(entryPrefix + i, cd.getPath().toUri().getPath());
assertEquals(pool, cd.getPool());
assertEquals(expiry.getTime(), cd.getExpiration().getMillis());
}
assertFalse("Unexpected # of cache directives found", dit.hasNext());
long nextId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path("/foobar")).setPool(pool).build());
assertEquals(prevId + 1, nextId);
} finally {
if (secondary != null) {
secondary.shutdown();
}
}
}
Aggregations