Examples with CreateFileContext alluxio.master.file.contexts.CreateFileContext used on opensource projects

Example 1 with CreateFileContext

use of alluxio.master.file.contexts.CreateFileContext in project alluxio by Alluxio.

the class InodeTree method createPath.

/**
 * Creates a file or directory at path.
 *
 * This method expects that the last edge leading to the target inode to be write-locked. If the
 * last existing inode in the path is /a/b/c and we want to create /a/b/c/d/e, the c->d edge must
 * be write locked.
 *
 * On success, createPath attempts to push the write lock forward as far as possible. For the
 * above example, createPath would take a write lock on d->e, and downgrade the c->d lock from a
 * write lock to a read lock. This may not be possible if inodePath is a composite path which
 * doesn't own the write lock. In that case no downgrade will occur.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path
 * @param context method context
 * @return a list of created inodes
 * @throws FileAlreadyExistsException when there is already a file at path if we want to create a
 *         directory there
 * @throws BlockInfoException when blockSizeBytes is invalid
 * @throws InvalidPathException when path is invalid, for example, (1) when there is nonexistent
 *         necessary parent directories and recursive is false, (2) when one of the necessary
 *         parent directories is actually a file
 * @throws FileDoesNotExistException if the parent of the path does not exist and the recursive
 *         option is false
 */
public List<Inode> createPath(RpcContext rpcContext, LockedInodePath inodePath, CreatePathContext<?, ?> context) throws FileAlreadyExistsException, BlockInfoException, InvalidPathException, IOException, FileDoesNotExistException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    // TODO(gpang): consider splitting this into createFilePath and createDirectoryPath, with a
    // helper method for the shared logic.
    AlluxioURI path = inodePath.getUri();
    if (path.isRoot()) {
        String errorMessage = "Not allowed to create existing root path: " + path;
        LOG.error(errorMessage);
        throw new FileAlreadyExistsException(errorMessage);
    }
    if (inodePath.fullPathExists()) {
        if (context instanceof CreateDirectoryContext && ((CreateDirectoryContext) context).getOptions().getAllowExists()) {
            return Collections.emptyList();
        } else {
            String pathType = "file";
            if (context instanceof CreateDirectoryContext) {
                pathType = "directory";
            }
            String errorMessage = String.format("Not allowed to create %s because path already exists: %s", pathType, path);
            throw new FileAlreadyExistsException(errorMessage);
        }
    }
    if (context instanceof CreateFileContext) {
        CreateFileContext fileContext = (CreateFileContext) context;
        if (fileContext.getOptions().getBlockSizeBytes() < 1) {
            throw new BlockInfoException("Invalid block size " + fileContext.getOptions().getBlockSizeBytes());
        }
    }
    LOG.debug("createPath {}", path);
    String[] pathComponents = inodePath.mPathComponents;
    String name = path.getName();
    // pathIndex is the index into pathComponents where we start filling in the path from the inode.
    int pathIndex = inodePath.getExistingInodeCount();
    if (pathIndex < pathComponents.length - 1) {
        // Otherwise we add the remaining path components to the list of components to create.
        if (!context.isRecursive()) {
            throw new FileDoesNotExistException(String.format("File %s creation failed. Component %d(%s) does not exist", path, pathIndex, pathComponents[pathIndex]));
        }
    }
    // The ancestor inode (parent or ancestor) of the target path.
    Inode ancestorInode = inodePath.getAncestorInode();
    if (!ancestorInode.isDirectory()) {
        throw new InvalidPathException("Could not traverse to parent directory of path " + path + ". Component " + pathComponents[pathIndex - 1] + " is not a directory.");
    }
    InodeDirectoryView currentInodeDirectory = ancestorInode.asDirectory();
    List<Inode> createdInodes = new ArrayList<>();
    if (context.isPersisted()) {
        // Synchronously persist directories. These inodes are already READ locked.
        for (Inode inode : inodePath.getInodeList()) {
            if (!inode.isPersisted()) {
                // This cast is safe because we've already verified that the file inode doesn't exist.
                syncPersistExistingDirectory(rpcContext, inode.asDirectory());
            }
        }
    }
    if ((pathIndex < (pathComponents.length - 1) || !mInodeStore.getChild(currentInodeDirectory, name).isPresent()) && context.getOperationTimeMs() > currentInodeDirectory.getLastModificationTimeMs()) {
        // (1) There are components in parent paths that need to be created. Or
        // (2) The last component of the path needs to be created.
        // In these two cases, the last traversed Inode will be modified if the new timestamp is after
        // the existing last modified time.
        long currentId = currentInodeDirectory.getId();
        try (LockResource lr = mInodeLockManager.lockUpdate(currentId)) {
            long updatedLastModified = mInodeStore.get(currentId).get().getLastModificationTimeMs();
            if (updatedLastModified < context.getOperationTimeMs()) {
                UpdateInodeEntry.Builder updateInodeEntry = UpdateInodeEntry.newBuilder().setId(currentId).setLastModificationTimeMs(context.getOperationTimeMs()).setLastAccessTimeMs(context.getOperationTimeMs());
                if (context.getXAttr() != null) {
                    updateInodeEntry.putAllXAttr(CommonUtils.convertToByteString(context.getXAttr()));
                }
                mState.applyAndJournal(rpcContext, updateInodeEntry.build());
            }
        }
    }
    // Fill in the ancestor directories that were missing.
    // NOTE, we set the mode of missing ancestor directories to be the default value, rather
    // than inheriting the option of the final file to create, because it may not have
    // "execute" permission.
    CreateDirectoryContext missingDirContext = CreateDirectoryContext.defaults();
    missingDirContext.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getTtl()).setTtlAction(context.getTtlAction()));
    missingDirContext.setWriteType(context.getWriteType());
    missingDirContext.setOperationTimeMs(context.getOperationTimeMs());
    missingDirContext.setMountPoint(false);
    missingDirContext.setOwner(context.getOwner());
    missingDirContext.setGroup(context.getGroup());
    missingDirContext.setXAttr(context.getXAttr());
    StringBuilder pathBuilder = new StringBuilder().append(String.join(AlluxioURI.SEPARATOR, Arrays.asList(pathComponents).subList(0, pathIndex)));
    for (int k = pathIndex; k < (pathComponents.length - 1); k++) {
        MutableInodeDirectory newDir = MutableInodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(rpcContext.getJournalContext()), currentInodeDirectory.getId(), pathComponents[k], missingDirContext);
        if (currentInodeDirectory.isPinned() && !newDir.isPinned()) {
            newDir.setPinned(true);
            newDir.setMediumTypes(new HashSet<>(currentInodeDirectory.getMediumTypes()));
        }
        inheritOwnerAndGroupIfEmpty(newDir, currentInodeDirectory);
        // if the parent has default ACL, copy that default ACL as the new directory's default
        // and access acl, ANDed with the umask
        // if it is part of a metadata load operation, we ignore the umask and simply inherit
        // the default ACL as the directory's new default and access ACL
        short mode = context.isMetadataLoad() ? Mode.createFullAccess().toShort() : newDir.getMode();
        DefaultAccessControlList dAcl = currentInodeDirectory.getDefaultACL();
        if (!dAcl.isEmpty()) {
            Pair<AccessControlList, DefaultAccessControlList> pair = dAcl.generateChildDirACL(mode);
            newDir.setInternalAcl(pair.getFirst());
            newDir.setDefaultACL(pair.getSecond());
        }
        String newDirPath = k == 0 ? ROOT_PATH : pathBuilder.append(AlluxioURI.SEPARATOR).append(pathComponents[k]).toString();
        mState.applyAndJournal(rpcContext, newDir, newDirPath);
        inodePath.addNextInode(Inode.wrap(newDir));
        // concurrent creates from trying to persist the same directory name.
        if (context.isPersisted()) {
            syncPersistExistingDirectory(rpcContext, newDir);
        }
        createdInodes.add(Inode.wrap(newDir));
        currentInodeDirectory = newDir;
    }
    // Create the final path component.
    MutableInode<?> newInode;
    // create the new inode, with a write lock
    if (context instanceof CreateDirectoryContext) {
        CreateDirectoryContext directoryContext = (CreateDirectoryContext) context;
        MutableInodeDirectory newDir = MutableInodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(rpcContext.getJournalContext()), currentInodeDirectory.getId(), name, directoryContext);
        // if the parent has default ACL, take the default ACL ANDed with the umask as the new
        // directory's default and access acl
        // When it is a metadata load operation, do not take the umask into account
        short mode = context.isMetadataLoad() ? Mode.createFullAccess().toShort() : newDir.getMode();
        DefaultAccessControlList dAcl = currentInodeDirectory.getDefaultACL();
        if (!dAcl.isEmpty()) {
            Pair<AccessControlList, DefaultAccessControlList> pair = dAcl.generateChildDirACL(mode);
            newDir.setInternalAcl(pair.getFirst());
            newDir.setDefaultACL(pair.getSecond());
        }
        if (directoryContext.isPersisted()) {
            // Do not journal the persist entry, since a creation entry will be journaled instead.
            if (context.isMetadataLoad()) {
                // if we are creating the file as a result of loading metadata, the newDir is already
                // persisted, and we got the permissions info from the ufs.
                newDir.setOwner(context.getOwner().intern()).setGroup(context.getGroup().intern()).setMode(context.getMode().toShort());
                Long operationTimeMs = context.getOperationTimeMs();
                if (operationTimeMs != null) {
                    newDir.setLastModificationTimeMs(operationTimeMs, true);
                    newDir.setLastAccessTimeMs(operationTimeMs, true);
                }
                newDir.setPersistenceState(PersistenceState.PERSISTED);
            } else {
                syncPersistNewDirectory(newDir);
            }
        }
        // Do NOT call setOwner/Group after inheriting from parent if empty
        inheritOwnerAndGroupIfEmpty(newDir, currentInodeDirectory);
        newInode = newDir;
    } else if (context instanceof CreateFileContext) {
        CreateFileContext fileContext = (CreateFileContext) context;
        MutableInodeFile newFile = MutableInodeFile.create(mContainerIdGenerator.getNewContainerId(), currentInodeDirectory.getId(), name, System.currentTimeMillis(), fileContext);
        // if the parent has a default ACL, copy that default ACL ANDed with the umask as the new
        // file's access ACL.
        // If it is a metadata load operation, do not consider the umask.
        DefaultAccessControlList dAcl = currentInodeDirectory.getDefaultACL();
        short mode = context.isMetadataLoad() ? Mode.createFullAccess().toShort() : newFile.getMode();
        if (!dAcl.isEmpty()) {
            AccessControlList acl = dAcl.generateChildFileACL(mode);
            newFile.setInternalAcl(acl);
        }
        if (fileContext.isCacheable()) {
            newFile.setCacheable(true);
        }
        if (fileContext.getWriteType() == WriteType.ASYNC_THROUGH) {
            newFile.setPersistenceState(PersistenceState.TO_BE_PERSISTED);
        }
        // Do NOT call setOwner/Group after inheriting from parent if empty
        inheritOwnerAndGroupIfEmpty(newFile, currentInodeDirectory);
        newInode = newFile;
    } else {
        throw new IllegalStateException(String.format("Unrecognized create options: %s", context));
    }
    if (currentInodeDirectory.isPinned() && !newInode.isPinned()) {
        newInode.setPinned(true);
        newInode.setMediumTypes(new HashSet<>(currentInodeDirectory.getMediumTypes()));
    }
    mState.applyAndJournal(rpcContext, newInode, inodePath.getUri().getPath());
    Inode inode = Inode.wrap(newInode);
    inodePath.addNextInode(inode);
    createdInodes.add(inode);
    LOG.debug("createFile: File Created: {} parent: {}", newInode, currentInodeDirectory);
    return createdInodes;
}

Example 2 with CreateFileContext

use of alluxio.master.file.contexts.CreateFileContext in project alluxio by Alluxio.

the class FileSystemMasterTest method renameToNonExistentParent.

@Test
public void renameToNonExistentParent() throws Exception {
    CreateFileContext context = CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true));
    mFileSystemMaster.createFile(NESTED_URI, context);
    try {
        mFileSystemMaster.rename(NESTED_URI, new AlluxioURI("/testDNE/b"), RenameContext.defaults());
        fail("Rename to a non-existent parent path should not succeed.");
    } catch (FileDoesNotExistException e) {
    // Expected case.
    }
}

Example 3 with CreateFileContext

use of alluxio.master.file.contexts.CreateFileContext in project alluxio by Alluxio.

the class FileSystemMasterTest method setNoTtlForFileWithTtl.

/**
 * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for a file.
 */
@Test
public void setNoTtlForFileWithTtl() throws Exception {
    mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true)));
    CreateFileContext context = CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)).setBlockSizeBytes(Constants.KB).setRecursive(true));
    long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context).getFileId();
    // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be
    // deleted during next TTL check.
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.NO_TTL))));
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK);
    assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId());
}

Example 4 with CreateFileContext

use of alluxio.master.file.contexts.CreateFileContext in project alluxio by Alluxio.

the class FileSystemMasterTest method renameUnderNonexistingDir.

/**
 * Tests that an exception is thrown when trying to create a file in a non-existing directory
 * without setting the {@code recursive} flag.
 */
@Test
public void renameUnderNonexistingDir() throws Exception {
    mThrown.expect(FileDoesNotExistException.class);
    mThrown.expectMessage(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/nested/test"));
    CreateFileContext context = CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB));
    mFileSystemMaster.createFile(TEST_URI, context);
    // nested dir
    mFileSystemMaster.rename(TEST_URI, NESTED_FILE_URI, RenameContext.defaults());
}

Example 5 with CreateFileContext

use of alluxio.master.file.contexts.CreateFileContext in project alluxio by Alluxio.

the class FileSystemMasterTest method setLargerTtlForFileWithTtl.

/**
 * Tests that a file has not been deleted after the TTL has been reset to a valid value.
 */
@Test
public void setLargerTtlForFileWithTtl() throws Exception {
    mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true)));
    CreateFileContext context = CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)).setBlockSizeBytes(Constants.KB).setRecursive(true));
    long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context).getFileId();
    assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId());
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS))));
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK);
    // TTL is reset to 1 hour, the file should not be deleted during last TTL check.
    assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId());
}