Examples with InodeFile alluxio.master.file.meta.InodeFile used on opensource projects

Example 6 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method generateFileBlockInfo.

/**
   * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
   * the block, such as the file offset, and additional UFS locations for the block.
   *
   * @param inodePath the file the block is a part of
   * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
   * @return a new {@link FileBlockInfo} for the block
   * @throws InvalidPathException if the mount table is not able to resolve the file
   */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException {
    InodeFile file = inodePath.getInodeFile();
    FileBlockInfo fileBlockInfo = new FileBlockInfo();
    fileBlockInfo.setBlockInfo(blockInfo);
    fileBlockInfo.setUfsLocations(new ArrayList<String>());
    // The sequence number part of the block id is the block index.
    long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
    fileBlockInfo.setOffset(offset);
    if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
        // No alluxio locations, but there is a checkpoint in the under storage system. Add the
        // locations from the under storage system.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        String ufsUri = resolution.getUri().toString();
        UnderFileSystem ufs = resolution.getUfs();
        List<String> locs;
        try {
            locs = ufs.getFileLocations(ufsUri, FileLocationOptions.defaults().setOffset(fileBlockInfo.getOffset()));
        } catch (IOException e) {
            return fileBlockInfo;
        }
        if (locs != null) {
            for (String loc : locs) {
                fileBlockInfo.getUfsLocations().add(loc);
            }
        }
    }
    return fileBlockInfo;
}

Example 7 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method setAttributeInternal.

/**
   * @param inodePath the {@link LockedInodePath} to use
   * @param replayed whether the operation is a result of replaying the journal
   * @param opTimeMs the operation time (in milliseconds)
   * @param options the method options
   * @return list of inodes which were marked as persisted
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if the file path corresponding to the file id is invalid
   * @throws AccessControlException if failed to set permission
   */
private List<Inode<?>> setAttributeInternal(LockedInodePath inodePath, boolean replayed, long opTimeMs, SetAttributeOptions options) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    List<Inode<?>> persistedInodes = Collections.emptyList();
    Inode<?> inode = inodePath.getInode();
    if (options.getPinned() != null) {
        mInodeTree.setPinned(inodePath, options.getPinned(), opTimeMs);
        inode.setLastModificationTimeMs(opTimeMs);
    }
    if (options.getTtl() != null) {
        long ttl = options.getTtl();
        if (inode.getTtl() != ttl) {
            mTtlBuckets.remove(inode);
            inode.setTtl(ttl);
            mTtlBuckets.insert(inode);
            inode.setLastModificationTimeMs(opTimeMs);
            inode.setTtlAction(options.getTtlAction());
        }
    }
    if (options.getPersisted() != null) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(((InodeFile) inode).isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        InodeFile file = (InodeFile) inode;
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(options.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!file.isPersisted()) {
            file.setPersistenceState(PersistenceState.PERSISTED);
            persistedInodes = propagatePersistedInternal(inodePath, false);
            file.setLastModificationTimeMs(opTimeMs);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (options.getOwner() != null) || (options.getGroup() != null);
    boolean modeChanged = (options.getMode() != Constants.INVALID_MODE);
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && !replayed && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !((InodeFile) inode).isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            if (UnderFileSystemUtils.isObjectStorage(ufsUri)) {
                LOG.warn("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
            } else {
                UnderFileSystem ufs = resolution.getUfs();
                if (ownerGroupChanged) {
                    try {
                        String owner = options.getOwner() != null ? options.getOwner() : inode.getOwner();
                        String group = options.getGroup() != null ? options.getGroup() : inode.getGroup();
                        ufs.setOwner(ufsUri, owner, group);
                    } catch (IOException e) {
                        throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                    }
                }
                if (modeChanged) {
                    try {
                        ufs.setMode(ufsUri, options.getMode());
                    } catch (IOException e) {
                        throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                    }
                }
            }
        }
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (options.getOwner() != null) {
        inode.setOwner(options.getOwner());
    }
    if (options.getGroup() != null) {
        inode.setGroup(options.getGroup());
    }
    if (modeChanged) {
        inode.setMode(options.getMode());
    }
    return persistedInodes;
}

Example 8 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method completeFileAndJournal.

/**
   * Completes a file. After a file is completed, it cannot be written to.
   * <p>
   * Writes to the journal.
   *
   * @param inodePath the {@link LockedInodePath} to complete
   * @param options the method options
   * @param journalContext the journal context
   * @throws InvalidPathException if an invalid path is encountered
   * @throws FileDoesNotExistException if the file does not exist
   * @throws BlockInfoException if a block information exception is encountered
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws InvalidFileSizeException if an invalid file size is encountered
   */
private void completeFileAndJournal(LockedInodePath inodePath, CompleteFileOptions options, JournalContext journalContext) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException {
    Inode<?> inode = inodePath.getInode();
    if (!inode.isFile()) {
        throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
    }
    InodeFile fileInode = (InodeFile) inode;
    List<Long> blockIdList = fileInode.getBlockIds();
    List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
    if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
        throw new BlockInfoException("Cannot complete a file without all the blocks committed");
    }
    // Iterate over all file blocks committed to Alluxio, computing the length and verify that all
    // the blocks (except the last one) is the same size as the file block size.
    long inMemoryLength = 0;
    long fileBlockSize = fileInode.getBlockSizeBytes();
    for (int i = 0; i < blockInfoList.size(); i++) {
        BlockInfo blockInfo = blockInfoList.get(i);
        inMemoryLength += blockInfo.getLength();
        if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
            throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
        }
    }
    // If the file is persisted, its length is determined by UFS. Otherwise, its length is
    // determined by its memory footprint.
    long length = fileInode.isPersisted() ? options.getUfsLength() : inMemoryLength;
    completeFileInternal(fileInode.getBlockIds(), inodePath, length, options.getOperationTimeMs());
    CompleteFileEntry completeFileEntry = CompleteFileEntry.newBuilder().addAllBlockIds(fileInode.getBlockIds()).setId(inode.getId()).setLength(length).setOpTimeMs(options.getOperationTimeMs()).build();
    appendJournalEntry(JournalEntry.newBuilder().setCompleteFile(completeFileEntry).build(), journalContext);
}

Example 9 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method completeFileInternal.

/**
   * @param blockIds the block ids to use
   * @param inodePath the {@link LockedInodePath} to complete
   * @param length the length to use
   * @param opTimeMs the operation time (in milliseconds)
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if an invalid path is encountered
   * @throws InvalidFileSizeException if an invalid file size is encountered
   * @throws FileAlreadyCompletedException if the file has already been completed
   */
private void completeFileInternal(List<Long> blockIds, LockedInodePath inodePath, long length, long opTimeMs) throws FileDoesNotExistException, InvalidPathException, InvalidFileSizeException, FileAlreadyCompletedException {
    InodeFile inode = inodePath.getInodeFile();
    inode.setBlockIds(blockIds);
    inode.setLastModificationTimeMs(opTimeMs);
    inode.complete(length);
    if (inode.isPersisted()) {
        // Commit all the file blocks (without locations) so the metadata for the block exists.
        long currLength = length;
        for (long blockId : inode.getBlockIds()) {
            long blockSize = Math.min(currLength, inode.getBlockSizeBytes());
            mBlockMaster.commitBlockInUFS(blockId, blockSize);
            currLength -= blockSize;
        }
    }
    Metrics.FILES_COMPLETED.inc();
}

Example 10 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method getInMemoryPercentage.

/**
   * Gets the in-memory percentage of an Inode. For a file that has all blocks in memory, it returns
   * 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory.
   *
   * @param inode the inode
   * @return the in memory percentage
   */
private int getInMemoryPercentage(Inode<?> inode) {
    if (!inode.isFile()) {
        return 0;
    }
    InodeFile inodeFile = (InodeFile) inode;
    long length = inodeFile.getLength();
    if (length == 0) {
        return 100;
    }
    long inMemoryLength = 0;
    for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) {
        if (isInTopStorageTier(info)) {
            inMemoryLength += info.getLength();
        }
    }
    return (int) (inMemoryLength * 100 / length);
}