Examples with INode org.apache.hadoop.hdfs.server.namenode.INode used on opensource projects

Example 21 with INode

use of org.apache.hadoop.hdfs.server.namenode.INode in project hadoop by apache.

the class CacheReplicationMonitor method rescanCacheDirectives.

/**
   * Scan all CacheDirectives.  Use the information to figure out
   * what cache replication factor each block should have.
   */
private void rescanCacheDirectives() {
    FSDirectory fsDir = namesystem.getFSDirectory();
    final long now = new Date().getTime();
    for (CacheDirective directive : cacheManager.getCacheDirectives()) {
        scannedDirectives++;
        // Skip processing this entry if it has expired
        if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) {
            LOG.debug("Directive {}: the directive expired at {} (now = {})", directive.getId(), directive.getExpiryTime(), now);
            continue;
        }
        String path = directive.getPath();
        INode node;
        try {
            node = fsDir.getINode(path, DirOp.READ);
        } catch (IOException e) {
            // We don't cache through symlinks or invalid paths
            LOG.debug("Directive {}: Failed to resolve path {} ({})", directive.getId(), path, e.getMessage());
            continue;
        }
        if (node == null) {
            LOG.debug("Directive {}: No inode found at {}", directive.getId(), path);
        } else if (node.isDirectory()) {
            INodeDirectory dir = node.asDirectory();
            ReadOnlyList<INode> children = dir.getChildrenList(Snapshot.CURRENT_STATE_ID);
            for (INode child : children) {
                if (child.isFile()) {
                    rescanFile(directive, child.asFile());
                }
            }
        } else if (node.isFile()) {
            rescanFile(directive, node.asFile());
        } else {
            LOG.debug("Directive {}: ignoring non-directive, non-file inode {} ", directive.getId(), node);
        }
    }
}

Example 22 with INode

use of org.apache.hadoop.hdfs.server.namenode.INode in project hadoop by apache.

the class DirectorySnapshottableFeature method computeDiffRecursively.

/**
   * Recursively compute the difference between snapshots under a given
   * directory/file.
   * @param snapshotRoot The directory where snapshots were taken.
   * @param node The directory/file under which the diff is computed.
   * @param parentPath Relative path (corresponding to the snapshot root) of
   *                   the node's parent.
   * @param diffReport data structure used to store the diff.
   */
private void computeDiffRecursively(final INodeDirectory snapshotRoot, INode node, List<byte[]> parentPath, SnapshotDiffInfo diffReport) {
    final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.getFrom() : diffReport.getTo();
    final Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.getTo() : diffReport.getFrom();
    byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]);
    if (node.isDirectory()) {
        final ChildrenDiff diff = new ChildrenDiff();
        INodeDirectory dir = node.asDirectory();
        DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
        if (sf != null) {
            boolean change = sf.computeDiffBetweenSnapshots(earlierSnapshot, laterSnapshot, diff, dir);
            if (change) {
                diffReport.addDirDiff(dir, relativePath, diff);
            }
        }
        ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot.getId());
        for (INode child : children) {
            final byte[] name = child.getLocalNameBytes();
            boolean toProcess = diff.searchIndex(ListType.DELETED, name) < 0;
            if (!toProcess && child instanceof INodeReference.WithName) {
                byte[][] renameTargetPath = findRenameTargetPath(snapshotRoot, (WithName) child, laterSnapshot == null ? Snapshot.CURRENT_STATE_ID : laterSnapshot.getId());
                if (renameTargetPath != null) {
                    toProcess = true;
                    diffReport.setRenameTarget(child.getId(), renameTargetPath);
                }
            }
            if (toProcess) {
                parentPath.add(name);
                computeDiffRecursively(snapshotRoot, child, parentPath, diffReport);
                parentPath.remove(parentPath.size() - 1);
            }
        }
    } else if (node.isFile() && node.asFile().isWithSnapshot()) {
        INodeFile file = node.asFile();
        boolean change = file.getFileWithSnapshotFeature().changedBetweenSnapshots(file, earlierSnapshot, laterSnapshot);
        if (change) {
            diffReport.addFileDiff(file, relativePath);
        }
    }
}

Example 23 with INode

use of org.apache.hadoop.hdfs.server.namenode.INode in project hadoop by apache.

the class DirectorySnapshottableFeature method findRenameTargetPath.

/**
   * We just found a deleted WithName node as the source of a rename operation.
   * However, we should include it in our snapshot diff report as rename only
   * if the rename target is also under the same snapshottable directory.
   */
private byte[][] findRenameTargetPath(final INodeDirectory snapshotRoot, INodeReference.WithName wn, final int snapshotId) {
    INode inode = wn.getReferredINode();
    final LinkedList<byte[]> ancestors = Lists.newLinkedList();
    while (inode != null) {
        if (inode == snapshotRoot) {
            return ancestors.toArray(new byte[ancestors.size()][]);
        }
        if (inode instanceof INodeReference.WithCount) {
            inode = ((WithCount) inode).getParentRef(snapshotId);
        } else {
            INode parent = inode.getParentReference() != null ? inode.getParentReference() : inode.getParent();
            if (parent != null && parent instanceof INodeDirectory) {
                int sid = parent.asDirectory().searchChild(inode);
                if (sid < snapshotId) {
                    return null;
                }
            }
            if (!(parent instanceof WithCount)) {
                ancestors.addFirst(inode.getLocalNameBytes());
            }
            inode = parent;
        }
    }
    return null;
}

Example 24 with INode

use of org.apache.hadoop.hdfs.server.namenode.INode in project hadoop by apache.

the class DirectoryWithSnapshotFeature method cleanDirectory.

public void cleanDirectory(INode.ReclaimContext reclaimContext, final INodeDirectory currentINode, final int snapshot, int prior) {
    Map<INode, INode> priorCreated = null;
    Map<INode, INode> priorDeleted = null;
    QuotaCounts old = reclaimContext.quotaDelta().getCountsCopy();
    if (snapshot == Snapshot.CURRENT_STATE_ID) {
        // delete the current directory
        currentINode.recordModification(prior);
        // delete everything in created list
        DirectoryDiff lastDiff = diffs.getLast();
        if (lastDiff != null) {
            lastDiff.diff.destroyCreatedList(reclaimContext, currentINode);
        }
        currentINode.cleanSubtreeRecursively(reclaimContext, snapshot, prior, null);
    } else {
        // update prior
        prior = getDiffs().updatePrior(snapshot, prior);
        // its original created and deleted list before deleting post
        if (prior != NO_SNAPSHOT_ID) {
            DirectoryDiff priorDiff = this.getDiffs().getDiffById(prior);
            if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
                List<INode> cList = priorDiff.diff.getList(ListType.CREATED);
                List<INode> dList = priorDiff.diff.getList(ListType.DELETED);
                priorCreated = cloneDiffList(cList);
                priorDeleted = cloneDiffList(dList);
            }
        }
        getDiffs().deleteSnapshotDiff(reclaimContext, snapshot, prior, currentINode);
        currentINode.cleanSubtreeRecursively(reclaimContext, snapshot, prior, priorDeleted);
        // check priorDiff again since it may be created during the diff deletion
        if (prior != NO_SNAPSHOT_ID) {
            DirectoryDiff priorDiff = this.getDiffs().getDiffById(prior);
            if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
                // cleanSubtreeRecursively call.
                if (priorCreated != null) {
                    // we only check the node originally in prior's created list
                    for (INode cNode : priorDiff.getChildrenDiff().getList(ListType.CREATED)) {
                        if (priorCreated.containsKey(cNode)) {
                            cNode.cleanSubtree(reclaimContext, snapshot, NO_SNAPSHOT_ID);
                        }
                    }
                }
                for (INode dNode : priorDiff.getChildrenDiff().getList(ListType.DELETED)) {
                    if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
                        cleanDeletedINode(reclaimContext, dNode, snapshot, prior);
                    }
                }
            }
        }
    }
    QuotaCounts current = reclaimContext.quotaDelta().getCountsCopy();
    current.subtract(old);
    if (currentINode.isQuotaSet()) {
        reclaimContext.quotaDelta().addQuotaDirUpdate(currentINode, current);
    }
}

Example 25 with INode

use of org.apache.hadoop.hdfs.server.namenode.INode in project hadoop by apache.

the class DirectoryWithSnapshotFeature method cleanDeletedINode.

/**
   * Clean an inode while we move it from the deleted list of post to the
   * deleted list of prior.
   * @param reclaimContext blocks and inodes that need to be reclaimed
   * @param inode The inode to clean.
   * @param post The post snapshot.
   * @param prior The id of the prior snapshot.
   */
private static void cleanDeletedINode(INode.ReclaimContext reclaimContext, INode inode, final int post, final int prior) {
    Deque<INode> queue = new ArrayDeque<>();
    queue.addLast(inode);
    while (!queue.isEmpty()) {
        INode topNode = queue.pollFirst();
        if (topNode instanceof INodeReference.WithName) {
            INodeReference.WithName wn = (INodeReference.WithName) topNode;
            if (wn.getLastSnapshotId() >= post) {
                INodeReference.WithCount wc = (INodeReference.WithCount) wn.getReferredINode();
                if (wc.getLastWithName() == wn && wc.getParentReference() == null) {
                    // this wn is the last wn inside of the wc, also the dstRef node has
                    // been deleted. In this case, we should treat the referred file/dir
                    // as normal case
                    queue.add(wc.getReferredINode());
                } else {
                    wn.cleanSubtree(reclaimContext, post, prior);
                }
            }
        // For DstReference node, since the node is not in the created list of
        // prior, we should treat it as regular file/dir
        } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
            INodeFile file = topNode.asFile();
            file.getDiffs().deleteSnapshotDiff(reclaimContext, post, prior, file);
        } else if (topNode.isDirectory()) {
            INodeDirectory dir = topNode.asDirectory();
            ChildrenDiff priorChildrenDiff = null;
            DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
            if (sf != null) {
                // delete files/dirs created after prior. Note that these
                // files/dirs, along with inode, were deleted right after post.
                DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior);
                if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
                    priorChildrenDiff = priorDiff.getChildrenDiff();
                    priorChildrenDiff.destroyCreatedList(reclaimContext, dir);
                }
            }
            for (INode child : dir.getChildrenList(prior)) {
                if (priorChildrenDiff != null && priorChildrenDiff.search(ListType.DELETED, child.getLocalNameBytes()) != null) {
                    continue;
                }
                queue.addLast(child);
            }
        }
    }
}