Examples with InodeFile alluxio.master.file.meta.InodeFile used on opensource projects

Example 16 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class FileSystemMaster method getFileInfoInternal.

/**
   * @param inodePath the {@link LockedInodePath} to get the {@link FileInfo} for
   * @return the {@link FileInfo} for the given inode
   * @throws FileDoesNotExistException if the file does not exist
   * @throws AccessControlException if permission denied
   */
private FileInfo getFileInfoInternal(LockedInodePath inodePath) throws FileDoesNotExistException, AccessControlException {
    Inode<?> inode = inodePath.getInode();
    AlluxioURI uri = inodePath.getUri();
    FileInfo fileInfo = inode.generateClientFileInfo(uri.toString());
    fileInfo.setInMemoryPercentage(getInMemoryPercentage(inode));
    if (inode instanceof InodeFile) {
        try {
            fileInfo.setFileBlockInfos(getFileBlockInfoListInternal(inodePath));
        } catch (InvalidPathException e) {
            throw new FileDoesNotExistException(e.getMessage(), e);
        }
    }
    MountTable.Resolution resolution;
    try {
        resolution = mMountTable.resolve(uri);
    } catch (InvalidPathException e) {
        throw new FileDoesNotExistException(e.getMessage(), e);
    }
    AlluxioURI resolvedUri = resolution.getUri();
    // Only set the UFS path if the path is nested under a mount point.
    if (!uri.equals(resolvedUri)) {
        fileInfo.setUfsPath(resolvedUri.toString());
    }
    Metrics.FILE_INFOS_GOT.inc();
    return fileInfo;
}

Example 17 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class ReplicationChecker method check.

private Set<Long> check(Set<Long> inodes, ReplicationHandler handler, Mode mode) throws InterruptedException {
    Set<Long> processedFileIds = new HashSet<>();
    for (long inodeId : inodes) {
        if (mActiveJobToInodeID.size() >= mMaxActiveJobs) {
            return processedFileIds;
        }
        if (mActiveJobToInodeID.containsValue(inodeId)) {
            continue;
        }
        Set<Triple<AlluxioURI, Long, Integer>> requests = new HashSet<>();
        // Throw if interrupted.
        if (Thread.interrupted()) {
            throw new InterruptedException("ReplicationChecker interrupted.");
        }
        // locking the entire path but just the inode file since this access is read-only.
        try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(inodeId, LockPattern.READ)) {
            InodeFile file = inodePath.getInodeFile();
            for (long blockId : file.getBlockIds()) {
                BlockInfo blockInfo = null;
                try {
                    blockInfo = mBlockMaster.getBlockInfo(blockId);
                } catch (BlockInfoException e) {
                // Cannot find this block in Alluxio from BlockMaster, possibly persisted in UFS
                } catch (UnavailableException e) {
                    // The block master is not available, wait for the next heartbeat
                    LOG.warn("The block master is not available: {}", e.toString());
                    return processedFileIds;
                }
                int currentReplicas = (blockInfo == null) ? 0 : blockInfo.getLocations().size();
                switch(mode) {
                    case EVICT:
                        int maxReplicas = file.getReplicationMax();
                        if (file.getPersistenceState() == PersistenceState.TO_BE_PERSISTED && file.getReplicationDurable() > maxReplicas) {
                            maxReplicas = file.getReplicationDurable();
                        }
                        if (currentReplicas > maxReplicas) {
                            requests.add(new ImmutableTriple<>(inodePath.getUri(), blockId, currentReplicas - maxReplicas));
                        }
                        break;
                    case REPLICATE:
                        int minReplicas = file.getReplicationMin();
                        if (file.getPersistenceState() == PersistenceState.TO_BE_PERSISTED && file.getReplicationDurable() > minReplicas) {
                            minReplicas = file.getReplicationDurable();
                        }
                        if (currentReplicas < minReplicas) {
                            // if this file is not persisted and block master thinks it is lost, no effort made
                            if (!file.isPersisted() && mBlockMaster.isBlockLost(blockId)) {
                                continue;
                            }
                            requests.add(new ImmutableTriple<>(inodePath.getUri(), blockId, minReplicas - currentReplicas));
                        }
                        break;
                    default:
                        LOG.warn("Unexpected replication mode {}.", mode);
                }
            }
        } catch (FileDoesNotExistException e) {
            LOG.warn("Failed to check replication level for inode id {} : {}", inodeId, e.toString());
        }
        for (Triple<AlluxioURI, Long, Integer> entry : requests) {
            AlluxioURI uri = entry.getLeft();
            long blockId = entry.getMiddle();
            int numReplicas = entry.getRight();
            try {
                long jobId;
                switch(mode) {
                    case EVICT:
                        jobId = handler.evict(uri, blockId, numReplicas);
                        break;
                    case REPLICATE:
                        jobId = handler.replicate(uri, blockId, numReplicas);
                        break;
                    default:
                        throw new RuntimeException(String.format("Unexpected replication mode {}.", mode));
                }
                processedFileIds.add(inodeId);
                mActiveJobToInodeID.put(jobId, inodeId);
            } catch (JobDoesNotExistException | ResourceExhaustedException e) {
                LOG.warn("The job service is busy, will retry later. {}", e.toString());
                return processedFileIds;
            } catch (UnavailableException e) {
                LOG.warn("Unable to complete the replication check: {}, will retry later.", e.toString());
                return processedFileIds;
            } catch (Exception e) {
                SAMPLING_LOG.warn("Unexpected exception encountered when starting a {} job (uri={}," + " block ID={}, num replicas={}) : {}", mode, uri, blockId, numReplicas, e.toString());
                LOG.debug("Job service unexpected exception: ", e);
            }
        }
    }
    return processedFileIds;
}

Example 18 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class ReplicationChecker method checkMisreplicated.

private void checkMisreplicated(Set<Long> inodes, ReplicationHandler handler) throws InterruptedException {
    for (long inodeId : inodes) {
        if (mActiveJobToInodeID.size() >= mMaxActiveJobs) {
            return;
        }
        if (mActiveJobToInodeID.containsValue(inodeId)) {
            continue;
        }
        // Throw if interrupted.
        if (Thread.interrupted()) {
            throw new InterruptedException("ReplicationChecker interrupted.");
        }
        try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(inodeId, LockPattern.READ)) {
            InodeFile file = inodePath.getInodeFile();
            for (long blockId : file.getBlockIds()) {
                BlockInfo blockInfo = null;
                try {
                    blockInfo = mBlockMaster.getBlockInfo(blockId);
                } catch (BlockInfoException e) {
                // Cannot find this block in Alluxio from BlockMaster, possibly persisted in UFS
                } catch (UnavailableException e) {
                    // The block master is not available, wait for the next heartbeat
                    LOG.warn("The block master is not available: {}", e.toString());
                    return;
                }
                if (blockInfo == null) {
                    // no block info available, we simply log and return;
                    LOG.warn("Block info is null");
                    return;
                }
                for (Map.Entry<String, String> entry : findMisplacedBlock(file, blockInfo).entrySet()) {
                    try {
                        final long jobId = handler.migrate(inodePath.getUri(), blockId, entry.getKey(), entry.getValue());
                        mActiveJobToInodeID.put(jobId, inodeId);
                    } catch (Exception e) {
                        LOG.warn("Unexpected exception encountered when starting a migration job (uri={}," + " block ID={}, workerHost= {}) : {}", inodePath.getUri(), blockId, entry.getKey(), e.toString());
                        LOG.debug("Exception: ", e);
                    }
                }
            }
        } catch (FileDoesNotExistException e) {
            LOG.warn("Failed to check replication level for inode id {} : {}", inodeId, e.toString());
        }
    }
}

Example 19 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class DefaultFileSystemMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    if (isPrimary) {
        LOG.info("Starting fs master as primary");
        InodeDirectory root = mInodeTree.getRoot();
        if (root == null) {
            try (JournalContext context = createJournalContext()) {
                mInodeTree.initializeRoot(SecurityUtils.getOwner(mMasterContext.getUserState()), SecurityUtils.getGroup(mMasterContext.getUserState(), ServerConfiguration.global()), ModeUtils.applyDirectoryUMask(Mode.createFullAccess(), ServerConfiguration.getString(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)), context);
            }
        } else if (!ServerConfiguration.getBoolean(PropertyKey.MASTER_SKIP_ROOT_ACL_CHECK)) {
            // For backwards-compatibility:
            // Empty root owner indicates that previously the master had no security. In this case, the
            // master is allowed to be started with security turned on.
            String serverOwner = SecurityUtils.getOwner(mMasterContext.getUserState());
            if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && !root.getOwner().isEmpty() && !root.getOwner().equals(serverOwner)) {
                // user is not the previous owner
                throw new PermissionDeniedException(ExceptionMessage.PERMISSION_DENIED.getMessage(String.format("Unauthorized user on root. inode owner: %s current user: %s", root.getOwner(), serverOwner)));
            }
        }
        // Initialize the ufs manager from the mount table.
        for (String key : mMountTable.getMountTable().keySet()) {
            if (key.equals(MountTable.ROOT)) {
                continue;
            }
            MountInfo mountInfo = mMountTable.getMountTable().get(key);
            UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(ServerConfiguration.global()).createMountSpecificConf(mountInfo.getOptions().getPropertiesMap()).setReadOnly(mountInfo.getOptions().getReadOnly()).setShared(mountInfo.getOptions().getShared());
            mUfsManager.addMount(mountInfo.getMountId(), mountInfo.getUfsUri(), ufsConf);
        }
        // Startup Checks and Periodic Threads.
        // Rebuild the list of persist jobs (mPersistJobs) and map of pending persist requests
        // (mPersistRequests)
        long persistInitialIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS);
        long persistMaxIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS);
        long persistMaxWaitMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS);
        for (Long id : mInodeTree.getToBePersistedIds()) {
            Inode inode = mInodeStore.get(id).get();
            if (inode.isDirectory() || // When file is completed it is added to persist reqs
            !inode.asFile().isCompleted() || inode.getPersistenceState() != PersistenceState.TO_BE_PERSISTED || inode.asFile().getShouldPersistTime() == Constants.NO_AUTO_PERSIST) {
                continue;
            }
            InodeFile inodeFile = inode.asFile();
            if (inodeFile.getPersistJobId() == Constants.PERSISTENCE_INVALID_JOB_ID) {
                mPersistRequests.put(inodeFile.getId(), new alluxio.time.ExponentialTimer(persistInitialIntervalMs, persistMaxIntervalMs, getPersistenceWaitTime(inodeFile.getShouldPersistTime()), persistMaxWaitMs));
            } else {
                AlluxioURI path;
                try {
                    path = mInodeTree.getPath(inodeFile);
                } catch (FileDoesNotExistException e) {
                    LOG.error("Failed to determine path for inode with id {}", id, e);
                    continue;
                }
                addPersistJob(id, inodeFile.getPersistJobId(), getPersistenceWaitTime(inodeFile.getShouldPersistTime()), path, inodeFile.getTempUfsPath());
            }
        }
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED)) {
            validateInodeBlocks(true);
        }
        int blockIntegrityCheckInterval = (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_INTERVAL);
        if (blockIntegrityCheckInterval > 0) {
            // negative or zero interval implies disabled
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_BLOCK_INTEGRITY_CHECK, new BlockIntegrityChecker(this), blockIntegrityCheckInterval, ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_TTL_CHECK, new InodeTtlChecker(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_FILES_DETECTION, new LostFileDetector(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        mReplicationCheckHeartbeatThread = new HeartbeatThread(HeartbeatContext.MASTER_REPLICATION_CHECK, new alluxio.master.file.replication.ReplicationChecker(mInodeTree, mBlockMaster, mSafeModeManager, mJobMasterClientPool), (int) ServerConfiguration.getMs(PropertyKey.MASTER_REPLICATION_CHECK_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState());
        ReconfigurableRegistry.register(this);
        getExecutorService().submit(mReplicationCheckHeartbeatThread);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER, new PersistenceScheduler(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        mPersistCheckerPool = new java.util.concurrent.ThreadPoolExecutor(PERSIST_CHECKER_POOL_THREADS, PERSIST_CHECKER_POOL_THREADS, 1, java.util.concurrent.TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), alluxio.util.ThreadFactoryUtils.build("Persist-Checker-%d", true));
        mPersistCheckerPool.allowCoreThreadTimeOut(true);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_CHECKER, new PersistenceChecker(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_TIME_SERIES, new TimeSeriesRecorder(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_METRICS_TIME_SERIES_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED)) {
            mAsyncAuditLogWriter = new AsyncUserAccessAuditLogWriter("AUDIT_LOG");
            mAsyncAuditLogWriter.start();
            MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_AUDIT_LOG_ENTRIES_SIZE.getName(), () -> mAsyncAuditLogWriter != null ? mAsyncAuditLogWriter.getAuditLogEntriesSize() : -1);
        }
        if (ServerConfiguration.getBoolean(PropertyKey.UNDERFS_CLEANUP_ENABLED)) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UFS_CLEANUP, new UfsCleaner(this), (int) ServerConfiguration.getMs(PropertyKey.UNDERFS_CLEANUP_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        mAccessTimeUpdater.start();
        mSyncManager.start();
    }
}

Example 20 with InodeFile

use of alluxio.master.file.meta.InodeFile in project alluxio by Alluxio.

the class DefaultFileSystemMaster method generateFileBlockInfo.

/**
 * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
 * the block, such as the file offset, and additional UFS locations for the block.
 *
 * @param inodePath the file the block is a part of
 * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
 * @return a new {@link FileBlockInfo} for the block
 */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws FileDoesNotExistException {
    InodeFile file = inodePath.getInodeFile();
    FileBlockInfo fileBlockInfo = new FileBlockInfo();
    fileBlockInfo.setBlockInfo(blockInfo);
    fileBlockInfo.setUfsLocations(new ArrayList<>());
    // The sequence number part of the block id is the block index.
    long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
    fileBlockInfo.setOffset(offset);
    if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
        // No alluxio locations, but there is a checkpoint in the under storage system. Add the
        // locations from the under storage system.
        long blockId = fileBlockInfo.getBlockInfo().getBlockId();
        List<String> locations = mUfsBlockLocationCache.get(blockId, inodePath.getUri(), fileBlockInfo.getOffset());
        if (locations != null) {
            fileBlockInfo.setUfsLocations(locations);
        }
    }
    return fileBlockInfo;
}