use of alluxio.underfs.Fingerprint in project alluxio by Alluxio.
the class DefaultFileSystemMaster method setAttributeSingleFile.
/**
* @param inodePath the {@link LockedInodePath} to use
* @param updateUfs whether to update the UFS with the attribute change
* @param opTimeMs the operation time (in milliseconds)
* @param context the method context
*/
protected void setAttributeSingleFile(RpcContext rpcContext, LockedInodePath inodePath, boolean updateUfs, long opTimeMs, SetAttributeContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
Inode inode = inodePath.getInode();
SetAttributePOptions.Builder protoOptions = context.getOptions();
if (protoOptions.hasPinned()) {
mInodeTree.setPinned(rpcContext, inodePath, context.getOptions().getPinned(), context.getOptions().getPinnedMediaList(), opTimeMs);
}
UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(inode.getId());
if (protoOptions.hasReplicationMax() || protoOptions.hasReplicationMin()) {
Integer replicationMax = protoOptions.hasReplicationMax() ? protoOptions.getReplicationMax() : null;
Integer replicationMin = protoOptions.hasReplicationMin() ? protoOptions.getReplicationMin() : null;
mInodeTree.setReplication(rpcContext, inodePath, replicationMax, replicationMin, opTimeMs);
}
// protoOptions may not have both fields set
if (protoOptions.hasCommonOptions()) {
FileSystemMasterCommonPOptions commonOpts = protoOptions.getCommonOptions();
TtlAction action = commonOpts.hasTtlAction() ? commonOpts.getTtlAction() : null;
Long ttl = commonOpts.hasTtl() ? commonOpts.getTtl() : null;
boolean modified = false;
if (ttl != null && inode.getTtl() != ttl) {
entry.setTtl(ttl);
modified = true;
}
if (action != null && inode.getTtlAction() != action) {
entry.setTtlAction(ProtobufUtils.toProtobuf(action));
modified = true;
}
if (modified) {
entry.setLastModificationTimeMs(opTimeMs);
}
}
if (protoOptions.hasPersisted()) {
Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
Preconditions.checkArgument(inode.asFile().isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
// TODO(manugoyal) figure out valid behavior in the un-persist case
Preconditions.checkArgument(protoOptions.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
if (!inode.asFile().isPersisted()) {
entry.setPersistenceState(PersistenceState.PERSISTED.name());
entry.setLastModificationTimeMs(context.getOperationTimeMs());
propagatePersistedInternal(rpcContext, inodePath);
Metrics.FILES_PERSISTED.inc();
}
}
boolean ownerGroupChanged = (protoOptions.hasOwner()) || (protoOptions.hasGroup());
boolean modeChanged = protoOptions.hasMode();
// If the file is persisted in UFS, also update corresponding owner/group/permission.
if ((ownerGroupChanged || modeChanged) && updateUfs && inode.isPersisted()) {
if ((inode instanceof InodeFile) && !inode.asFile().isCompleted()) {
LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
} else {
checkUfsMode(inodePath.getUri(), OperationType.WRITE);
MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
String ufsUri = resolution.getUri().toString();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
if (ufs.isObjectStorage()) {
LOG.debug("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
} else {
String owner = null;
String group = null;
String mode = null;
if (ownerGroupChanged) {
try {
owner = protoOptions.getOwner() != null ? protoOptions.getOwner() : inode.getOwner();
group = protoOptions.getGroup() != null ? protoOptions.getGroup() : inode.getGroup();
ufs.setOwner(ufsUri, owner, group);
} catch (IOException e) {
throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
}
}
if (modeChanged) {
try {
mode = String.valueOf(protoOptions.getMode());
ufs.setMode(ufsUri, ModeUtils.protoToShort(protoOptions.getMode()));
} catch (IOException e) {
throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
}
}
// Retrieve the ufs fingerprint after the ufs changes.
String existingFingerprint = inode.getUfsFingerprint();
if (!existingFingerprint.equals(Constants.INVALID_UFS_FINGERPRINT)) {
// Update existing fingerprint, since contents did not change
Fingerprint fp = Fingerprint.parse(existingFingerprint);
fp.putTag(Fingerprint.Tag.OWNER, owner);
fp.putTag(Fingerprint.Tag.GROUP, group);
fp.putTag(Fingerprint.Tag.MODE, mode);
context.setUfsFingerprint(fp.serialize());
} else {
// Need to retrieve the fingerprint from ufs.
context.setUfsFingerprint(ufs.getFingerprint(ufsUri));
}
}
}
}
}
if (!context.getUfsFingerprint().equals(Constants.INVALID_UFS_FINGERPRINT)) {
entry.setUfsFingerprint(context.getUfsFingerprint());
}
// Only commit the set permission to inode after the propagation to UFS succeeded.
if (protoOptions.hasOwner()) {
entry.setOwner(protoOptions.getOwner());
}
if (protoOptions.hasGroup()) {
entry.setGroup(protoOptions.getGroup());
}
if (modeChanged) {
entry.setMode(ModeUtils.protoToShort(protoOptions.getMode()));
}
mInodeTree.updateInode(rpcContext, entry.build());
}
use of alluxio.underfs.Fingerprint in project alluxio by Alluxio.
the class DefaultFileSystemMaster method completeFileInternal.
/**
* Completes a file. After a file is completed, it cannot be written to.
*
* @param rpcContext the rpc context
* @param inodePath the {@link LockedInodePath} to complete
* @param context the method context
*/
void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePath, CompleteFileContext context) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, UnavailableException {
Inode inode = inodePath.getInode();
if (!inode.isFile()) {
throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
}
InodeFile fileInode = inode.asFile();
List<Long> blockIdList = fileInode.getBlockIds();
List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
throw new BlockInfoException("Cannot complete a file without all the blocks committed");
}
// Iterate over all file blocks committed to Alluxio, computing the length and verify that all
// the blocks (except the last one) is the same size as the file block size.
long inAlluxioLength = 0;
long fileBlockSize = fileInode.getBlockSizeBytes();
for (int i = 0; i < blockInfoList.size(); i++) {
BlockInfo blockInfo = blockInfoList.get(i);
inAlluxioLength += blockInfo.getLength();
if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
}
}
// If the file is persisted, its length is determined by UFS. Otherwise, its length is
// determined by its size in Alluxio.
long length = fileInode.isPersisted() ? context.getOptions().getUfsLength() : inAlluxioLength;
String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
if (fileInode.isPersisted()) {
UfsStatus ufsStatus = context.getUfsStatus();
// Retrieve the UFS fingerprint for this file.
MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
AlluxioURI resolvedUri = resolution.getUri();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
if (ufsStatus == null) {
ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
} else {
ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
}
}
}
completeFileInternal(rpcContext, inodePath, length, context.getOperationTimeMs(), ufsFingerprint);
}
use of alluxio.underfs.Fingerprint in project alluxio by Alluxio.
the class InodeSyncStream method syncExistingInodeMetadata.
/**
* Sync inode metadata with the UFS state.
*
* This method expects the {@code inodePath} to already exist in the inode tree.
*/
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
}
// Set to true if the given inode was deleted.
boolean deletedInode = false;
// whether we need to load metadata for the current path
boolean loadMetadata = mLoadOnly;
LOG.trace("Syncing inode metadata {}", inodePath.getUri());
// The requested path already exists in Alluxio.
Inode inode = inodePath.getInode();
// initialize sync children to true if it is a listStatus call on a directory
boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
// Do not sync an incomplete file, since the UFS file is expected to not exist.
return;
}
Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
if (!persistingLock.isPresent()) {
// written.
return;
}
persistingLock.get().close();
UfsStatus cachedStatus = null;
boolean fileNotFound = false;
try {
cachedStatus = mStatusCache.getStatus(inodePath.getUri());
} catch (FileNotFoundException e) {
fileNotFound = true;
}
MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
AlluxioURI ufsUri = resolution.getUri();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
String ufsFingerprint;
Fingerprint ufsFpParsed;
// When the status is not cached and it was not due to file not found, we retry
if (fileNotFound) {
ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else if (cachedStatus == null) {
// TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else {
// When the status is cached
Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
} else {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
}
ufsFingerprint = ufsFpParsed.serialize();
}
boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
if (syncPlan.toUpdateMetaData()) {
// It works by calling SetAttributeInternal on the inodePath.
if (ufsFpParsed != null && ufsFpParsed.isValid()) {
short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
long opTimeMs = System.currentTimeMillis();
SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
if (!owner.equals(Fingerprint.UNDERSCORE)) {
// Only set owner if not empty
builder.setOwner(owner);
}
String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
if (!group.equals(Fingerprint.UNDERSCORE)) {
// Only set group if not empty
builder.setGroup(group);
}
SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
}
}
if (syncPlan.toDelete()) {
deletedInode = true;
try {
// The options for deleting.
DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
} catch (DirectoryNotEmptyException | IOException e) {
// Should not happen, since it is an unchecked delete.
LOG.error("Unexpected error for unchecked delete.", e);
}
}
if (syncPlan.toLoadMetadata()) {
loadMetadata = true;
}
syncChildren = syncPlan.toSyncChildren();
}
}
// if DescendantType.ONE we only sync children if we are syncing root of this stream
if (mDescendantType == DescendantType.ONE) {
syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
}
Map<String, Inode> inodeChildren = new HashMap<>();
if (syncChildren) {
// maps children name to inode
mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
// Fetch and populate children into the cache
mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
// Iterate over UFS listings and process UFS children.
if (listStatus != null) {
for (UfsStatus ufsChildStatus : listStatus) {
if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
// Ufs child exists, but Alluxio child does not. Must load metadata.
loadMetadata = true;
break;
}
}
}
}
// locked path
if (deletedInode) {
inodePath.removeLastInode();
}
// load metadata if necessary.
if (loadMetadata && !skipLoad) {
loadMetadataForPath(inodePath);
}
if (syncChildren) {
// Iterate over Alluxio children and process persisted children.
mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
// its children.
if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
return;
}
// If we're performing a recursive sync, add each child of our current Inode to the queue
AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
mPendingPaths.add(child);
// This asynchronously schedules a job to pre-fetch the statuses into the cache.
if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
mStatusCache.prefetchChildren(child, mMountTable);
}
});
}
}
use of alluxio.underfs.Fingerprint in project alluxio by Alluxio.
the class UfsSyncUtils method computeSyncPlan.
/**
* Given an inode and ufs status, returns a sync plan describing how to
* sync the inode with the ufs.
*
* @param inode the inode to sync
* @param ufsFingerprint the ufs fingerprint to check for the sync
* @param containsMountPoint true if this inode contains a mount point, false otherwise
* @return a {@link SyncPlan} describing how to sync the inode with the ufs
*/
public static SyncPlan computeSyncPlan(Inode inode, Fingerprint ufsFingerprint, boolean containsMountPoint) {
Fingerprint inodeFingerprint = Fingerprint.parse(inode.getUfsFingerprint());
boolean isContentSynced = inodeUfsIsContentSynced(inode, inodeFingerprint, ufsFingerprint);
boolean isMetadataSynced = inodeUfsIsMetadataSynced(inode, inodeFingerprint, ufsFingerprint);
boolean ufsExists = ufsFingerprint.isValid();
boolean ufsIsDir = ufsFingerprint != null && Fingerprint.Type.DIRECTORY.name().equals(ufsFingerprint.getTag(Fingerprint.Tag.TYPE));
UfsSyncUtils.SyncPlan syncPlan = new UfsSyncUtils.SyncPlan();
if (isContentSynced && isMetadataSynced) {
// Inode is already synced.
if (inode.isDirectory() && inode.isPersisted()) {
// Both Alluxio and UFS are directories, so sync the children of the directory.
syncPlan.setSyncChildren();
}
return syncPlan;
}
// One of the metadata or content is not in sync
if (inode.isDirectory() && (containsMountPoint || ufsIsDir)) {
// - directory permissions can be updated without removing the inode
if (inode.getParentId() != InodeTree.NO_PARENT) {
// Only update the inode if it is not the root directory. The root directory is a special
// case, since it is expected to be owned by the process that starts the master, and not
// the owner on UFS.
syncPlan.setUpdateMetadata();
}
syncPlan.setSyncChildren();
return syncPlan;
}
// a directory. That requires a deletion and reload as well.
if (!isContentSynced) {
// update inode, by deleting and then optionally loading metadata
syncPlan.setDelete();
if (ufsExists) {
// UFS exists, so load metadata later.
syncPlan.setLoadMetadata();
}
} else {
syncPlan.setUpdateMetadata();
}
return syncPlan;
}
use of alluxio.underfs.Fingerprint in project alluxio by Alluxio.
the class DefaultFileSystemMaster method completeFileInternal.
/**
* @param rpcContext the rpc context
* @param inodePath the {@link LockedInodePath} to complete
* @param length the length to use
* @param opTimeMs the operation time (in milliseconds)
* @param ufsFingerprint the ufs fingerprint
*/
private void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePath, long length, long opTimeMs, String ufsFingerprint) throws FileDoesNotExistException, InvalidPathException, InvalidFileSizeException, FileAlreadyCompletedException, UnavailableException {
Preconditions.checkState(inodePath.getLockPattern().isWrite());
InodeFile inode = inodePath.getInodeFile();
if (inode.isCompleted() && inode.getLength() != Constants.UNKNOWN_SIZE) {
throw new FileAlreadyCompletedException(String.format("File %s has already been completed.", inode.getName()));
}
if (length < 0 && length != Constants.UNKNOWN_SIZE) {
throw new InvalidFileSizeException("File " + inode.getName() + " cannot have negative length: " + length);
}
Builder entry = UpdateInodeFileEntry.newBuilder().setId(inode.getId()).setPath(inodePath.getUri().getPath()).setCompleted(true).setLength(length);
if (length == Constants.UNKNOWN_SIZE) {
// TODO(gpang): allow unknown files to be multiple blocks.
// If the length of the file is unknown, only allow 1 block to the file.
length = inode.getBlockSizeBytes();
}
int sequenceNumber = 0;
long remainingBytes = length;
while (remainingBytes > 0) {
entry.addSetBlocks(BlockId.createBlockId(inode.getBlockContainerId(), sequenceNumber));
remainingBytes -= Math.min(remainingBytes, inode.getBlockSizeBytes());
sequenceNumber++;
}
if (inode.isPersisted()) {
// Commit all the file blocks (without locations) so the metadata for the block exists.
commitBlockInfosForFile(entry.getSetBlocksList(), length, inode.getBlockSizeBytes());
// The path exists in UFS, so it is no longer absent
mUfsAbsentPathCache.processExisting(inodePath.getUri());
}
// We could introduce a concept of composite entries, so that these two entries could
// be applied in a single call to applyAndJournal.
mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inode.getId()).setUfsFingerprint(ufsFingerprint).setLastModificationTimeMs(opTimeMs).setLastAccessTimeMs(opTimeMs).setOverwriteModificationTime(true).build());
mInodeTree.updateInodeFile(rpcContext, entry.build());
Metrics.FILES_COMPLETED.inc();
}
Aggregations