use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.
the class InodeSyncStream method syncExistingInodeMetadata.
/**
* Sync inode metadata with the UFS state.
*
* This method expects the {@code inodePath} to already exist in the inode tree.
*/
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
}
// Set to true if the given inode was deleted.
boolean deletedInode = false;
// whether we need to load metadata for the current path
boolean loadMetadata = mLoadOnly;
LOG.trace("Syncing inode metadata {}", inodePath.getUri());
// The requested path already exists in Alluxio.
Inode inode = inodePath.getInode();
// initialize sync children to true if it is a listStatus call on a directory
boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
// Do not sync an incomplete file, since the UFS file is expected to not exist.
return;
}
Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
if (!persistingLock.isPresent()) {
// written.
return;
}
persistingLock.get().close();
UfsStatus cachedStatus = null;
boolean fileNotFound = false;
try {
cachedStatus = mStatusCache.getStatus(inodePath.getUri());
} catch (FileNotFoundException e) {
fileNotFound = true;
}
MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
AlluxioURI ufsUri = resolution.getUri();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
String ufsFingerprint;
Fingerprint ufsFpParsed;
// When the status is not cached and it was not due to file not found, we retry
if (fileNotFound) {
ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else if (cachedStatus == null) {
// TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else {
// When the status is cached
Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
} else {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
}
ufsFingerprint = ufsFpParsed.serialize();
}
boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
if (syncPlan.toUpdateMetaData()) {
// It works by calling SetAttributeInternal on the inodePath.
if (ufsFpParsed != null && ufsFpParsed.isValid()) {
short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
long opTimeMs = System.currentTimeMillis();
SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
if (!owner.equals(Fingerprint.UNDERSCORE)) {
// Only set owner if not empty
builder.setOwner(owner);
}
String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
if (!group.equals(Fingerprint.UNDERSCORE)) {
// Only set group if not empty
builder.setGroup(group);
}
SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
}
}
if (syncPlan.toDelete()) {
deletedInode = true;
try {
// The options for deleting.
DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
} catch (DirectoryNotEmptyException | IOException e) {
// Should not happen, since it is an unchecked delete.
LOG.error("Unexpected error for unchecked delete.", e);
}
}
if (syncPlan.toLoadMetadata()) {
loadMetadata = true;
}
syncChildren = syncPlan.toSyncChildren();
}
}
// if DescendantType.ONE we only sync children if we are syncing root of this stream
if (mDescendantType == DescendantType.ONE) {
syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
}
Map<String, Inode> inodeChildren = new HashMap<>();
if (syncChildren) {
// maps children name to inode
mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
// Fetch and populate children into the cache
mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
// Iterate over UFS listings and process UFS children.
if (listStatus != null) {
for (UfsStatus ufsChildStatus : listStatus) {
if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
// Ufs child exists, but Alluxio child does not. Must load metadata.
loadMetadata = true;
break;
}
}
}
}
// locked path
if (deletedInode) {
inodePath.removeLastInode();
}
// load metadata if necessary.
if (loadMetadata && !skipLoad) {
loadMetadataForPath(inodePath);
}
if (syncChildren) {
// Iterate over Alluxio children and process persisted children.
mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
// its children.
if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
return;
}
// If we're performing a recursive sync, add each child of our current Inode to the queue
AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
mPendingPaths.add(child);
// This asynchronously schedules a job to pre-fetch the statuses into the cache.
if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
mStatusCache.prefetchChildren(child, mMountTable);
}
});
}
}
use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.
the class InodeTtlChecker method heartbeat.
@Override
public void heartbeat() throws InterruptedException {
Set<TtlBucket> expiredBuckets = mTtlBuckets.getExpiredBuckets(System.currentTimeMillis());
for (TtlBucket bucket : expiredBuckets) {
for (Inode inode : bucket.getInodes()) {
// Throw if interrupted.
if (Thread.interrupted()) {
throw new InterruptedException("InodeTtlChecker interrupted.");
}
AlluxioURI path = null;
try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(inode.getId(), LockPattern.READ)) {
path = inodePath.getUri();
} catch (FileDoesNotExistException e) {
// The inode has already been deleted, nothing needs to be done.
continue;
} catch (Exception e) {
LOG.error("Exception trying to clean up {} for ttl check: {}", inode.toString(), e.toString());
}
if (path != null) {
try {
TtlAction ttlAction = inode.getTtlAction();
LOG.info("Path {} TTL has expired, performing action {}", path.getPath(), ttlAction);
switch(ttlAction) {
case FREE:
// parent of file
if (inode.isDirectory()) {
mFileSystemMaster.free(path, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true).setRecursive(true)));
} else {
mFileSystemMaster.free(path, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true)));
}
try (JournalContext journalContext = mFileSystemMaster.createJournalContext()) {
// Reset state
mInodeTree.updateInode(journalContext, UpdateInodeEntry.newBuilder().setId(inode.getId()).setTtl(Constants.NO_TTL).setTtlAction(ProtobufUtils.toProtobuf(TtlAction.DELETE)).build());
}
mTtlBuckets.remove(inode);
break;
case // Default if not set is DELETE
DELETE:
// parent of file
if (inode.isDirectory()) {
mFileSystemMaster.delete(path, DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
} else {
mFileSystemMaster.delete(path, DeleteContext.defaults());
}
break;
default:
LOG.error("Unknown ttl action {}", ttlAction);
}
} catch (Exception e) {
LOG.error("Exception trying to clean up {} for ttl check", inode.toString(), e);
}
}
}
}
mTtlBuckets.removeBuckets(expiredBuckets);
}
use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.
the class AccessTimeUpdaterTest method createInode.
private void createInode(String path, CreateFileContext context) throws Exception {
try (LockedInodePath inodePath = mInodeTree.lockInodePath(new AlluxioURI(path), InodeTree.LockPattern.WRITE_EDGE)) {
List<Inode> result = mInodeTree.createPath(RpcContext.NOOP, inodePath, context);
MutableInode<?> inode = mInodeStore.getMutable(result.get(result.size() - 1).getId()).get();
mInodeStore.writeInode(inode);
}
}
use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.
the class ReplicationCheckerTest method createBlockHelper.
/**
* Helper to create a file with a single block.
*
* @param path Alluxio path of the file
* @param context context to create the file
* @param pinLocation
* @return the block ID
*/
private long createBlockHelper(AlluxioURI path, CreatePathContext<?, ?> context, String pinLocation) throws Exception {
try (LockedInodePath inodePath = mInodeTree.lockInodePath(path, LockPattern.WRITE_EDGE)) {
List<Inode> created = mInodeTree.createPath(RpcContext.NOOP, inodePath, context);
if (!pinLocation.equals("")) {
mInodeTree.setPinned(RpcContext.NOOP, inodePath, true, ImmutableList.of(pinLocation), 0);
}
MutableInodeFile inodeFile = mInodeStore.getMutable(created.get(0).getId()).get().asFile();
inodeFile.setBlockSizeBytes(1);
inodeFile.setBlockIds(Arrays.asList(inodeFile.getNewBlockId()));
inodeFile.setCompleted(true);
mInodeStore.writeInode(inodeFile);
return inodeFile.getBlockIdByIndex(0);
}
}
use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.
the class InodeStoreTest method addRemoveAddList.
@Test
public void addRemoveAddList() {
writeInode(mRoot);
for (int i = 1; i < 10; i++) {
MutableInodeFile file = inodeFile(i, 0, "file" + i);
writeInode(file);
writeEdge(mRoot, file);
}
assertEquals(9, Iterables.size(mStore.getChildren(mRoot)));
for (Inode child : mStore.getChildren(mRoot)) {
MutableInode<?> childMut = mStore.getMutable(child.getId()).get();
removeParentEdge(childMut);
removeInode(childMut);
}
for (int i = 1; i < 10; i++) {
MutableInodeFile file = inodeFile(i, 0, "file" + i);
writeInode(file);
writeEdge(mRoot, file);
}
assertEquals(9, Iterables.size(mStore.getChildren(mRoot)));
}
Aggregations