use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.
the class InodeTree method createPath.
/**
* Creates a file or directory at path.
*
* @param inodePath the path
* @param options method options
* @return a {@link CreatePathResult} representing the modified inodes and created inodes during
* path creation
* @throws FileAlreadyExistsException when there is already a file at path if we want to create a
* directory there
* @throws BlockInfoException when blockSizeBytes is invalid
* @throws InvalidPathException when path is invalid, for example, (1) when there is nonexistent
* necessary parent directories and recursive is false, (2) when one of the necessary
* parent directories is actually a file
* @throws IOException if creating the path fails
* @throws FileDoesNotExistException if the parent of the path does not exist and the recursive
* option is false
*/
public CreatePathResult createPath(LockedInodePath inodePath, CreatePathOptions<?> options) throws FileAlreadyExistsException, BlockInfoException, InvalidPathException, IOException, FileDoesNotExistException {
AlluxioURI path = inodePath.getUri();
if (path.isRoot()) {
String errorMessage = ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(path);
LOG.error(errorMessage);
throw new FileAlreadyExistsException(errorMessage);
}
if (options instanceof CreateFileOptions) {
CreateFileOptions fileOptions = (CreateFileOptions) options;
if (fileOptions.getBlockSizeBytes() < 1) {
throw new BlockInfoException("Invalid block size " + fileOptions.getBlockSizeBytes());
}
}
if (!(inodePath instanceof MutableLockedInodePath)) {
throw new InvalidPathException(ExceptionMessage.NOT_MUTABLE_INODE_PATH.getMessage(inodePath.getUri()));
}
LOG.debug("createPath {}", path);
TraversalResult traversalResult = traverseToInode(inodePath, LockMode.WRITE_PARENT);
InodeLockList lockList = traversalResult.getInodeLockList();
MutableLockedInodePath extensibleInodePath = (MutableLockedInodePath) inodePath;
String[] pathComponents = extensibleInodePath.getPathComponents();
String name = path.getName();
// pathIndex is the index into pathComponents where we start filling in the path from the inode.
int pathIndex = extensibleInodePath.getInodes().size();
if (pathIndex < pathComponents.length - 1) {
// Otherwise we add the remaining path components to the list of components to create.
if (!options.isRecursive()) {
final String msg = new StringBuilder().append("File ").append(path).append(" creation failed. Component ").append(pathIndex).append("(").append(pathComponents[pathIndex]).append(") does not exist").toString();
LOG.error("FileDoesNotExistException: {}", msg);
throw new FileDoesNotExistException(msg);
}
}
// The ancestor inode (parent or ancestor) of the target path.
Inode<?> ancestorInode = extensibleInodePath.getAncestorInode();
if (!ancestorInode.isDirectory()) {
throw new InvalidPathException("Could not traverse to parent directory of path " + path + ". Component " + pathComponents[pathIndex - 1] + " is not a directory.");
}
InodeDirectory currentInodeDirectory = (InodeDirectory) ancestorInode;
List<Inode<?>> createdInodes = new ArrayList<>();
List<Inode<?>> modifiedInodes = new ArrayList<>();
// These are inodes that already exist, that should be journaled as persisted.
List<Inode<?>> existingNonPersisted = new ArrayList<>();
// These are inodes to mark as persisted at the end of this method.
List<Inode<?>> toPersistDirectories = new ArrayList<>();
if (options.isPersisted()) {
// Directory persistence will not happen until the end of this method.
toPersistDirectories.addAll(traversalResult.getNonPersisted());
existingNonPersisted.addAll(traversalResult.getNonPersisted());
}
if (pathIndex < (pathComponents.length - 1) || currentInodeDirectory.getChild(name) == null) {
// (1) There are components in parent paths that need to be created. Or
// (2) The last component of the path needs to be created.
// In these two cases, the last traversed Inode will be modified.
modifiedInodes.add(currentInodeDirectory);
}
// TODO(gpang): We may not have to lock the newly created inodes if the last inode is write
// locked. This could improve performance. Further investigation is needed.
// Fill in the ancestor directories that were missing.
// NOTE, we set the mode of missing ancestor directories to be the default value, rather
// than inheriting the option of the final file to create, because it may not have
// "execute" permission.
CreateDirectoryOptions missingDirOptions = CreateDirectoryOptions.defaults().setMountPoint(false).setPersisted(options.isPersisted()).setOwner(options.getOwner()).setGroup(options.getGroup());
for (int k = pathIndex; k < (pathComponents.length - 1); k++) {
InodeDirectory dir = InodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(), currentInodeDirectory.getId(), pathComponents[k], missingDirOptions);
// Lock the newly created inode before subsequent operations, and add it to the lock group.
lockList.lockWriteAndCheckNameAndParent(dir, currentInodeDirectory, pathComponents[k]);
dir.setPinned(currentInodeDirectory.isPinned());
currentInodeDirectory.addChild(dir);
currentInodeDirectory.setLastModificationTimeMs(options.getOperationTimeMs());
if (options.isPersisted()) {
toPersistDirectories.add(dir);
}
createdInodes.add(dir);
mInodes.add(dir);
currentInodeDirectory = dir;
}
// Create the final path component. First we need to make sure that there isn't already a file
// here with that name. If there is an existing file that is a directory and we're creating a
// directory, update persistence property of the directories if needed, otherwise, throw
// FileAlreadyExistsException unless options.allowExists is true.
Inode<?> lastInode = currentInodeDirectory.getChild(name);
if (lastInode != null) {
// Lock the last inode before subsequent operations, and add it to the lock group.
lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
if (lastInode.isDirectory() && options instanceof CreateDirectoryOptions && !lastInode.isPersisted() && options.isPersisted()) {
// The final path component already exists and is not persisted, so it should be added
// to the non-persisted Inodes of traversalResult.
existingNonPersisted.add(lastInode);
toPersistDirectories.add(lastInode);
} else if (!lastInode.isDirectory() || !(options instanceof CreateDirectoryOptions && ((CreateDirectoryOptions) options).isAllowExists())) {
String errorMessage = ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(path);
LOG.error(errorMessage);
throw new FileAlreadyExistsException(errorMessage);
}
} else {
if (options instanceof CreateDirectoryOptions) {
CreateDirectoryOptions directoryOptions = (CreateDirectoryOptions) options;
lastInode = InodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(), currentInodeDirectory.getId(), name, directoryOptions);
// Lock the created inode before subsequent operations, and add it to the lock group.
lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
if (directoryOptions.isPersisted()) {
toPersistDirectories.add(lastInode);
}
lastInode.setPinned(currentInodeDirectory.isPinned());
} else if (options instanceof CreateFileOptions) {
CreateFileOptions fileOptions = (CreateFileOptions) options;
lastInode = InodeFile.create(mContainerIdGenerator.getNewContainerId(), currentInodeDirectory.getId(), name, System.currentTimeMillis(), fileOptions);
// Lock the created inode before subsequent operations, and add it to the lock group.
lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
if (currentInodeDirectory.isPinned()) {
// Update set of pinned file ids.
mPinnedInodeFileIds.add(lastInode.getId());
}
lastInode.setPinned(currentInodeDirectory.isPinned());
}
createdInodes.add(lastInode);
mInodes.add(lastInode);
currentInodeDirectory.addChild(lastInode);
currentInodeDirectory.setLastModificationTimeMs(options.getOperationTimeMs());
}
// we mark it as persisted.
for (Inode<?> inode : toPersistDirectories) {
MountTable.Resolution resolution = mMountTable.resolve(getPath(inode));
String ufsUri = resolution.getUri().toString();
UnderFileSystem ufs = resolution.getUfs();
MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(inode.getOwner()).setGroup(inode.getGroup()).setMode(new Mode(inode.getMode()));
if (ufs.isDirectory(ufsUri) || ufs.mkdirs(ufsUri, mkdirsOptions)) {
inode.setPersistenceState(PersistenceState.PERSISTED);
}
}
// Extend the inodePath with the created inodes.
extensibleInodePath.getInodes().addAll(createdInodes);
LOG.debug("createFile: File Created: {} parent: {}", lastInode, currentInodeDirectory);
return new CreatePathResult(modifiedInodes, createdInodes, existingNonPersisted);
}
use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.
the class FileSystemMaster method renameInternal.
/**
* Implements renaming.
*
* @param srcInodePath the path of the rename source
* @param dstInodePath the path to the rename destination
* @param replayed whether the operation is a result of replaying the journal
* @param options method options
* @throws FileDoesNotExistException if a non-existent file is encountered
* @throws InvalidPathException if an invalid path is encountered
* @throws IOException if an I/O error is encountered
*/
private void renameInternal(LockedInodePath srcInodePath, LockedInodePath dstInodePath, boolean replayed, RenameOptions options) throws FileDoesNotExistException, InvalidPathException, IOException {
// Rename logic:
// 1. Change the source inode name to the destination name.
// 2. Insert the source inode into the destination parent.
// 3. Do UFS operations if necessary.
// 4. Remove the source inode (reverting the name) from the source parent.
// 5. Set the last modification times for both source and destination parent inodes.
Inode<?> srcInode = srcInodePath.getInode();
AlluxioURI srcPath = srcInodePath.getUri();
AlluxioURI dstPath = dstInodePath.getUri();
InodeDirectory srcParentInode = srcInodePath.getParentInodeDirectory();
InodeDirectory dstParentInode = dstInodePath.getParentInodeDirectory();
String srcName = srcPath.getName();
String dstName = dstPath.getName();
LOG.debug("Renaming {} to {}", srcPath, dstPath);
// 1. Change the source inode name to the destination name.
srcInode.setName(dstName);
srcInode.setParentId(dstParentInode.getId());
// 2. Insert the source inode into the destination parent.
if (!dstParentInode.addChild(srcInode)) {
// On failure, revert changes and throw exception.
srcInode.setName(srcName);
srcInode.setParentId(srcParentInode.getId());
throw new InvalidPathException("Destination path: " + dstPath + " already exists.");
}
// If the source file is persisted, rename it in the UFS.
try {
if (!replayed && srcInode.isPersisted()) {
MountTable.Resolution resolution = mMountTable.resolve(srcPath);
String ufsSrcPath = resolution.getUri().toString();
UnderFileSystem ufs = resolution.getUfs();
String ufsDstUri = mMountTable.resolve(dstPath).getUri().toString();
// Create ancestor directories from top to the bottom. We cannot use recursive create
// parents here because the permission for the ancestors can be different.
List<Inode<?>> dstInodeList = dstInodePath.getInodeList();
Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
AlluxioURI curUfsDirPath = new AlluxioURI(ufsDstUri).getParent();
// The dst inode does not exist yet, so the last inode in the list is the existing parent.
for (int i = dstInodeList.size() - 1; i >= 0; i--) {
if (ufs.isDirectory(curUfsDirPath.toString())) {
break;
}
Inode<?> curInode = dstInodeList.get(i);
MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(curInode.getOwner()).setGroup(curInode.getGroup()).setMode(new Mode(curInode.getMode()));
ufsDirsToMakeWithOptions.push(new Pair<>(curUfsDirPath.toString(), mkdirsOptions));
curUfsDirPath = curUfsDirPath.getParent();
}
while (!ufsDirsToMakeWithOptions.empty()) {
Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond())) {
throw new IOException(ExceptionMessage.FAILED_UFS_CREATE.getMessage(ufsDirAndPerm.getFirst()));
}
}
boolean success;
if (srcInode.isFile()) {
success = ufs.renameFile(ufsSrcPath, ufsDstUri);
} else {
success = ufs.renameDirectory(ufsSrcPath, ufsDstUri);
}
if (!success) {
throw new IOException(ExceptionMessage.FAILED_UFS_RENAME.getMessage(ufsSrcPath, ufsDstUri));
}
}
} catch (Exception e) {
// On failure, revert changes and throw exception.
if (!dstParentInode.removeChild(dstName)) {
LOG.error("Failed to revert rename changes. Alluxio metadata may be inconsistent.");
}
srcInode.setName(srcName);
srcInode.setParentId(srcParentInode.getId());
throw e;
}
// TODO(jiri): A crash between now and the time the rename operation is journaled will result in
// an inconsistency between Alluxio and UFS.
// 4. Remove the source inode (reverting the name) from the source parent. The name must be
// reverted or removeChild will not be able to find the appropriate child entry since it is
// keyed on the original name.
srcInode.setName(srcName);
if (!srcParentInode.removeChild(srcInode)) {
// This should never happen.
LOG.error("Failed to rename {} to {} in Alluxio. Alluxio and under storage may be " + "inconsistent.", srcPath, dstPath);
srcInode.setName(dstName);
if (!dstParentInode.removeChild(dstName)) {
LOG.error("Failed to revert changes when renaming {} to {}. Alluxio metadata may be " + "inconsistent.", srcPath, dstPath);
}
srcInode.setName(srcName);
srcInode.setParentId(srcParentInode.getId());
throw new IOException("Failed to remove source path " + srcPath + " from parent");
}
srcInode.setName(dstName);
// 5. Set the last modification times for both source and destination parent inodes.
// Note this step relies on setLastModificationTimeMs being thread safe to guarantee the
// correct behavior when multiple files are being renamed within a directory.
dstParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
srcParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
Metrics.PATHS_RENAMED.inc();
}
use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.
the class UnderFileSystemUtils method prepareFilePath.
/**
* Creates parent directories for path with correct permissions if required.
*
* @param alluxioPath Alluxio path
* @param ufsPath path in the under file system
* @param fs file system master client
* @param ufs the under file system
* @throws AlluxioException if an unexpected Alluxio exception is thrown
* @throws IOException if folder creation fails
*/
public static void prepareFilePath(AlluxioURI alluxioPath, String ufsPath, FileSystem fs, UnderFileSystem ufs) throws AlluxioException, IOException {
AlluxioURI dstPath = new AlluxioURI(ufsPath);
String parentPath = dstPath.getParent().getPath();
// creates the parent folder if it does not exist
if (!ufs.isDirectory(parentPath)) {
// Create ancestor directories from top to the bottom. We cannot use recursive create parents
// here because the permission for the ancestors can be different.
Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
AlluxioURI curAlluxioPath = alluxioPath.getParent();
AlluxioURI curUfsPath = dstPath.getParent();
// Stop at Alluxio mount point because the mapped directory in UFS may not exist.
while (curUfsPath != null && !ufs.isDirectory(curUfsPath.toString()) && curAlluxioPath != null) {
URIStatus curDirStatus = fs.getStatus(curAlluxioPath);
if (curDirStatus.isMountPoint()) {
throw new IOException(ExceptionMessage.UFS_PATH_DOES_NOT_EXIST.getMessage(curUfsPath));
}
ufsDirsToMakeWithOptions.push(new Pair<>(curUfsPath.toString(), MkdirsOptions.defaults().setCreateParent(false).setOwner(curDirStatus.getOwner()).setGroup(curDirStatus.getGroup()).setMode(new Mode((short) curDirStatus.getMode()))));
curAlluxioPath = curAlluxioPath.getParent();
curUfsPath = curUfsPath.getParent();
}
while (!ufsDirsToMakeWithOptions.empty()) {
Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
// and assume the directory is already prepared, regardless of permission matching.
if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond()) && !ufs.isDirectory(ufsDirAndPerm.getFirst())) {
throw new IOException("Failed to create dir: " + ufsDirAndPerm.getFirst());
}
}
}
}
Aggregations