Examples with Inode alluxio.master.file.meta.Inode used on opensource projects

Example 36 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method propagatePersistedInternal.

/**
 * Propagates the persisted status to all parents of the given inode in the same mount partition.
 *
 * @param journalContext the journal context
 * @param inodePath the inode to start the propagation at
 * @return list of inodes which were marked as persisted
 */
private void propagatePersistedInternal(Supplier<JournalContext> journalContext, LockedInodePath inodePath) throws FileDoesNotExistException {
    Inode inode = inodePath.getInode();
    List<Inode> inodes = inodePath.getInodeList();
    // Traverse the inodes from target inode to the root.
    Collections.reverse(inodes);
    // Skip the first, to not examine the target inode itself.
    inodes = inodes.subList(1, inodes.size());
    List<Inode> persistedInodes = new ArrayList<>();
    for (Inode ancestor : inodes) {
        // the path is already locked.
        AlluxioURI path = mInodeTree.getPath(ancestor);
        if (mMountTable.isMountPoint(path)) {
            // Stop propagating the persisted status at mount points.
            break;
        }
        if (ancestor.isPersisted()) {
            // Stop if a persisted directory is encountered.
            break;
        }
        mInodeTree.updateInode(journalContext, UpdateInodeEntry.newBuilder().setId(ancestor.getId()).setPersistenceState(PersistenceState.PERSISTED.name()).build());
    }
}

Example 37 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    if (isPrimary) {
        LOG.info("Starting fs master as primary");
        InodeDirectory root = mInodeTree.getRoot();
        if (root == null) {
            try (JournalContext context = createJournalContext()) {
                mInodeTree.initializeRoot(SecurityUtils.getOwner(mMasterContext.getUserState()), SecurityUtils.getGroup(mMasterContext.getUserState(), ServerConfiguration.global()), ModeUtils.applyDirectoryUMask(Mode.createFullAccess(), ServerConfiguration.getString(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)), context);
            }
        } else if (!ServerConfiguration.getBoolean(PropertyKey.MASTER_SKIP_ROOT_ACL_CHECK)) {
            // For backwards-compatibility:
            // Empty root owner indicates that previously the master had no security. In this case, the
            // master is allowed to be started with security turned on.
            String serverOwner = SecurityUtils.getOwner(mMasterContext.getUserState());
            if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && !root.getOwner().isEmpty() && !root.getOwner().equals(serverOwner)) {
                // user is not the previous owner
                throw new PermissionDeniedException(ExceptionMessage.PERMISSION_DENIED.getMessage(String.format("Unauthorized user on root. inode owner: %s current user: %s", root.getOwner(), serverOwner)));
            }
        }
        // Initialize the ufs manager from the mount table.
        for (String key : mMountTable.getMountTable().keySet()) {
            if (key.equals(MountTable.ROOT)) {
                continue;
            }
            MountInfo mountInfo = mMountTable.getMountTable().get(key);
            UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(ServerConfiguration.global()).createMountSpecificConf(mountInfo.getOptions().getPropertiesMap()).setReadOnly(mountInfo.getOptions().getReadOnly()).setShared(mountInfo.getOptions().getShared());
            mUfsManager.addMount(mountInfo.getMountId(), mountInfo.getUfsUri(), ufsConf);
        }
        // Startup Checks and Periodic Threads.
        // Rebuild the list of persist jobs (mPersistJobs) and map of pending persist requests
        // (mPersistRequests)
        long persistInitialIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS);
        long persistMaxIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS);
        long persistMaxWaitMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS);
        for (Long id : mInodeTree.getToBePersistedIds()) {
            Inode inode = mInodeStore.get(id).get();
            if (inode.isDirectory() || // When file is completed it is added to persist reqs
            !inode.asFile().isCompleted() || inode.getPersistenceState() != PersistenceState.TO_BE_PERSISTED || inode.asFile().getShouldPersistTime() == Constants.NO_AUTO_PERSIST) {
                continue;
            }
            InodeFile inodeFile = inode.asFile();
            if (inodeFile.getPersistJobId() == Constants.PERSISTENCE_INVALID_JOB_ID) {
                mPersistRequests.put(inodeFile.getId(), new alluxio.time.ExponentialTimer(persistInitialIntervalMs, persistMaxIntervalMs, getPersistenceWaitTime(inodeFile.getShouldPersistTime()), persistMaxWaitMs));
            } else {
                AlluxioURI path;
                try {
                    path = mInodeTree.getPath(inodeFile);
                } catch (FileDoesNotExistException e) {
                    LOG.error("Failed to determine path for inode with id {}", id, e);
                    continue;
                }
                addPersistJob(id, inodeFile.getPersistJobId(), getPersistenceWaitTime(inodeFile.getShouldPersistTime()), path, inodeFile.getTempUfsPath());
            }
        }
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED)) {
            validateInodeBlocks(true);
        }
        long blockIntegrityCheckInterval = ServerConfiguration.getMs(PropertyKey.MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_INTERVAL);
        if (blockIntegrityCheckInterval > 0) {
            // negative or zero interval implies disabled
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_BLOCK_INTEGRITY_CHECK, new BlockIntegrityChecker(this), blockIntegrityCheckInterval, ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_TTL_CHECK, new InodeTtlChecker(this, mInodeTree), ServerConfiguration.getMs(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_FILES_DETECTION, new LostFileDetector(this, mInodeTree), ServerConfiguration.getMs(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        mReplicationCheckHeartbeatThread = new HeartbeatThread(HeartbeatContext.MASTER_REPLICATION_CHECK, new alluxio.master.file.replication.ReplicationChecker(mInodeTree, mBlockMaster, mSafeModeManager, mJobMasterClientPool), ServerConfiguration.getMs(PropertyKey.MASTER_REPLICATION_CHECK_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState());
        ReconfigurableRegistry.register(this);
        getExecutorService().submit(mReplicationCheckHeartbeatThread);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER, new PersistenceScheduler(), ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        mPersistCheckerPool = new java.util.concurrent.ThreadPoolExecutor(PERSIST_CHECKER_POOL_THREADS, PERSIST_CHECKER_POOL_THREADS, 1, java.util.concurrent.TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), alluxio.util.ThreadFactoryUtils.build("Persist-Checker-%d", true));
        mPersistCheckerPool.allowCoreThreadTimeOut(true);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_CHECKER, new PersistenceChecker(), ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_TIME_SERIES, new TimeSeriesRecorder(), ServerConfiguration.getMs(PropertyKey.MASTER_METRICS_TIME_SERIES_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED)) {
            mAsyncAuditLogWriter = new AsyncUserAccessAuditLogWriter("AUDIT_LOG");
            mAsyncAuditLogWriter.start();
            MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_AUDIT_LOG_ENTRIES_SIZE.getName(), () -> mAsyncAuditLogWriter != null ? mAsyncAuditLogWriter.getAuditLogEntriesSize() : -1);
        }
        if (ServerConfiguration.getBoolean(PropertyKey.UNDERFS_CLEANUP_ENABLED)) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UFS_CLEANUP, new UfsCleaner(this), ServerConfiguration.getMs(PropertyKey.UNDERFS_CLEANUP_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        mAccessTimeUpdater.start();
        mSyncManager.start();
    }
}

Example 38 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setAclSingleInode.

private void setAclSingleInode(RpcContext rpcContext, SetAclAction action, LockedInodePath inodePath, List<AclEntry> entries, boolean replay, long opTimeMs) throws IOException, FileDoesNotExistException {
    Preconditions.checkState(inodePath.getLockPattern().isWrite());
    Inode inode = inodePath.getInode();
    // Check that we are not removing an extended mask.
    if (action == SetAclAction.REMOVE) {
        for (AclEntry entry : entries) {
            if ((entry.isDefault() && inode.getDefaultACL().hasExtended()) || (!entry.isDefault() && inode.getACL().hasExtended())) {
                if (entry.getType() == AclEntryType.MASK) {
                    throw new InvalidArgumentException("Deleting the mask for an extended ACL is not allowed. entry: " + entry);
                }
            }
        }
    }
    // Check that we are not setting default ACL to a file
    if (inode.isFile()) {
        for (AclEntry entry : entries) {
            if (entry.isDefault()) {
                throw new UnsupportedOperationException("Can not set default ACL for a file");
            }
        }
    }
    mInodeTree.setAcl(rpcContext, SetAclEntry.newBuilder().setId(inode.getId()).setOpTimeMs(opTimeMs).setAction(ProtoUtils.toProto(action)).addAllEntries(entries.stream().map(ProtoUtils::toProto).collect(Collectors.toList())).build());
    try {
        if (!replay && inode.isPersisted()) {
            setUfsAcl(inodePath);
        }
    } catch (InvalidPathException | AccessControlException e) {
        LOG.warn("Setting ufs ACL failed for path: {}", inodePath.getUri(), e);
    // TODO(david): revert the acl and default acl to the initial state if writing to ufs failed.
    }
}

Example 39 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getFileInfoInternal.

/**
 * @param inodePath the {@link LockedInodePath} to get the {@link FileInfo} for
 * @return the {@link FileInfo} for the given inode
 */
private FileInfo getFileInfoInternal(LockedInodePath inodePath, Counter counter) throws FileDoesNotExistException, UnavailableException {
    int inMemoryPercentage;
    int inAlluxioPercentage;
    Inode inode = inodePath.getInode();
    AlluxioURI uri = inodePath.getUri();
    FileInfo fileInfo = inode.generateClientFileInfo(uri.toString());
    if (fileInfo.isFolder()) {
        fileInfo.setLength(inode.asDirectory().getChildCount());
    }
    if (inode.isFile()) {
        InodeFile inodeFile = inode.asFile();
        List<BlockInfo> blockInfos = mBlockMaster.getBlockInfoList(inodeFile.getBlockIds());
        inMemoryPercentage = getFileInMemoryPercentageInternal(inodeFile, blockInfos);
        inAlluxioPercentage = getFileInAlluxioPercentageInternal(inodeFile, blockInfos);
        fileInfo.setInMemoryPercentage(inMemoryPercentage);
        fileInfo.setInAlluxioPercentage(inAlluxioPercentage);
        List<FileBlockInfo> fileBlockInfos = new ArrayList<>(blockInfos.size());
        for (BlockInfo blockInfo : blockInfos) {
            fileBlockInfos.add(generateFileBlockInfo(inodePath, blockInfo));
        }
        fileInfo.setFileBlockInfos(fileBlockInfos);
    }
    // Rehydrate missing block-infos for persisted files.
    if (fileInfo.isCompleted() && fileInfo.getBlockIds().size() > fileInfo.getFileBlockInfos().size() && inode.isPersisted()) {
        List<Long> missingBlockIds = fileInfo.getBlockIds().stream().filter((bId) -> fileInfo.getFileBlockInfo(bId) != null).collect(Collectors.toList());
        LOG.warn("BlockInfo missing for file: {}. BlockIdsWithMissingInfos: {}", inodePath.getUri(), missingBlockIds.stream().map(Object::toString).collect(Collectors.joining(",")));
        // Remove old block metadata from block-master before re-committing.
        mBlockMaster.removeBlocks(fileInfo.getBlockIds(), true);
        // Commit all the file blocks (without locations) so the metadata for the block exists.
        commitBlockInfosForFile(fileInfo.getBlockIds(), fileInfo.getLength(), fileInfo.getBlockSizeBytes());
        // Reset file-block-info list with the new list.
        try {
            fileInfo.setFileBlockInfos(getFileBlockInfoListInternal(inodePath));
        } catch (InvalidPathException e) {
            throw new FileDoesNotExistException(String.format("Hydration failed for file: %s", inodePath.getUri()), e);
        }
    }
    fileInfo.setXAttr(inode.getXAttr());
    MountTable.Resolution resolution;
    try {
        resolution = mMountTable.resolve(uri);
    } catch (InvalidPathException e) {
        throw new FileDoesNotExistException(e.getMessage(), e);
    }
    AlluxioURI resolvedUri = resolution.getUri();
    fileInfo.setUfsPath(resolvedUri.toString());
    fileInfo.setMountId(resolution.getMountId());
    if (counter == null) {
        Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.GET_FILE_INFO).inc();
    } else {
        counter.inc();
    }
    Metrics.FILE_INFOS_GOT.inc();
    return fileInfo;
}

Example 40 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method freeInternal.

/**
 * Implements free operation.
 *
 * @param rpcContext the rpc context
 * @param inodePath inode of the path to free
 * @param context context to free method
 */
private void freeInternal(RpcContext rpcContext, LockedInodePath inodePath, FreeContext context) throws FileDoesNotExistException, UnexpectedAlluxioException, IOException, InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInode();
    if (inode.isDirectory() && !context.getOptions().getRecursive() && mInodeStore.hasChildren(inode.asDirectory())) {
        // inode is nonempty, and we don't free a nonempty directory unless recursive is true
        throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_NON_EMPTY_DIR.getMessage(mInodeTree.getPath(inode)));
    }
    long opTimeMs = System.currentTimeMillis();
    List<Inode> freeInodes = new ArrayList<>();
    freeInodes.add(inode);
    try (LockedInodePathList descendants = mInodeTree.getDescendants(inodePath)) {
        for (LockedInodePath descedant : Iterables.concat(descendants, Collections.singleton(inodePath))) {
            Inode freeInode = descedant.getInodeOrNull();
            if (freeInode != null && freeInode.isFile()) {
                if (freeInode.getPersistenceState() != PersistenceState.PERSISTED) {
                    throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_NON_PERSISTED_FILE.getMessage(mInodeTree.getPath(freeInode)));
                }
                if (freeInode.isPinned()) {
                    if (!context.getOptions().getForced()) {
                        throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_PINNED_FILE.getMessage(mInodeTree.getPath(freeInode)));
                    }
                    SetAttributeContext setAttributeContext = SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setRecursive(false).setPinned(false));
                    setAttributeSingleFile(rpcContext, descedant, true, opTimeMs, setAttributeContext);
                }
                // Remove corresponding blocks from workers.
                mBlockMaster.removeBlocks(freeInode.asFile().getBlockIds(), false);
            }
        }
    }
    Metrics.FILES_FREED.inc(freeInodes.size());
}