Examples with UnderFileSystemConfiguration alluxio.underfs.UnderFileSystemConfiguration used on opensource projects

Example 21 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class DefaultFileSystemMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    if (isPrimary) {
        LOG.info("Starting fs master as primary");
        InodeDirectory root = mInodeTree.getRoot();
        if (root == null) {
            try (JournalContext context = createJournalContext()) {
                mInodeTree.initializeRoot(SecurityUtils.getOwner(mMasterContext.getUserState()), SecurityUtils.getGroup(mMasterContext.getUserState(), ServerConfiguration.global()), ModeUtils.applyDirectoryUMask(Mode.createFullAccess(), ServerConfiguration.getString(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)), context);
            }
        } else if (!ServerConfiguration.getBoolean(PropertyKey.MASTER_SKIP_ROOT_ACL_CHECK)) {
            // For backwards-compatibility:
            // Empty root owner indicates that previously the master had no security. In this case, the
            // master is allowed to be started with security turned on.
            String serverOwner = SecurityUtils.getOwner(mMasterContext.getUserState());
            if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && !root.getOwner().isEmpty() && !root.getOwner().equals(serverOwner)) {
                // user is not the previous owner
                throw new PermissionDeniedException(ExceptionMessage.PERMISSION_DENIED.getMessage(String.format("Unauthorized user on root. inode owner: %s current user: %s", root.getOwner(), serverOwner)));
            }
        }
        // Initialize the ufs manager from the mount table.
        for (String key : mMountTable.getMountTable().keySet()) {
            if (key.equals(MountTable.ROOT)) {
                continue;
            }
            MountInfo mountInfo = mMountTable.getMountTable().get(key);
            UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(ServerConfiguration.global()).createMountSpecificConf(mountInfo.getOptions().getPropertiesMap()).setReadOnly(mountInfo.getOptions().getReadOnly()).setShared(mountInfo.getOptions().getShared());
            mUfsManager.addMount(mountInfo.getMountId(), mountInfo.getUfsUri(), ufsConf);
        }
        // Startup Checks and Periodic Threads.
        // Rebuild the list of persist jobs (mPersistJobs) and map of pending persist requests
        // (mPersistRequests)
        long persistInitialIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS);
        long persistMaxIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS);
        long persistMaxWaitMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS);
        for (Long id : mInodeTree.getToBePersistedIds()) {
            Inode inode = mInodeStore.get(id).get();
            if (inode.isDirectory() || // When file is completed it is added to persist reqs
            !inode.asFile().isCompleted() || inode.getPersistenceState() != PersistenceState.TO_BE_PERSISTED || inode.asFile().getShouldPersistTime() == Constants.NO_AUTO_PERSIST) {
                continue;
            }
            InodeFile inodeFile = inode.asFile();
            if (inodeFile.getPersistJobId() == Constants.PERSISTENCE_INVALID_JOB_ID) {
                mPersistRequests.put(inodeFile.getId(), new alluxio.time.ExponentialTimer(persistInitialIntervalMs, persistMaxIntervalMs, getPersistenceWaitTime(inodeFile.getShouldPersistTime()), persistMaxWaitMs));
            } else {
                AlluxioURI path;
                try {
                    path = mInodeTree.getPath(inodeFile);
                } catch (FileDoesNotExistException e) {
                    LOG.error("Failed to determine path for inode with id {}", id, e);
                    continue;
                }
                addPersistJob(id, inodeFile.getPersistJobId(), getPersistenceWaitTime(inodeFile.getShouldPersistTime()), path, inodeFile.getTempUfsPath());
            }
        }
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED)) {
            validateInodeBlocks(true);
        }
        int blockIntegrityCheckInterval = (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_INTERVAL);
        if (blockIntegrityCheckInterval > 0) {
            // negative or zero interval implies disabled
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_BLOCK_INTEGRITY_CHECK, new BlockIntegrityChecker(this), blockIntegrityCheckInterval, ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_TTL_CHECK, new InodeTtlChecker(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_FILES_DETECTION, new LostFileDetector(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        mReplicationCheckHeartbeatThread = new HeartbeatThread(HeartbeatContext.MASTER_REPLICATION_CHECK, new alluxio.master.file.replication.ReplicationChecker(mInodeTree, mBlockMaster, mSafeModeManager, mJobMasterClientPool), (int) ServerConfiguration.getMs(PropertyKey.MASTER_REPLICATION_CHECK_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState());
        ReconfigurableRegistry.register(this);
        getExecutorService().submit(mReplicationCheckHeartbeatThread);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER, new PersistenceScheduler(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        mPersistCheckerPool = new java.util.concurrent.ThreadPoolExecutor(PERSIST_CHECKER_POOL_THREADS, PERSIST_CHECKER_POOL_THREADS, 1, java.util.concurrent.TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), alluxio.util.ThreadFactoryUtils.build("Persist-Checker-%d", true));
        mPersistCheckerPool.allowCoreThreadTimeOut(true);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_CHECKER, new PersistenceChecker(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_TIME_SERIES, new TimeSeriesRecorder(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_METRICS_TIME_SERIES_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED)) {
            mAsyncAuditLogWriter = new AsyncUserAccessAuditLogWriter("AUDIT_LOG");
            mAsyncAuditLogWriter.start();
            MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_AUDIT_LOG_ENTRIES_SIZE.getName(), () -> mAsyncAuditLogWriter != null ? mAsyncAuditLogWriter.getAuditLogEntriesSize() : -1);
        }
        if (ServerConfiguration.getBoolean(PropertyKey.UNDERFS_CLEANUP_ENABLED)) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UFS_CLEANUP, new UfsCleaner(this), (int) ServerConfiguration.getMs(PropertyKey.UNDERFS_CLEANUP_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        mAccessTimeUpdater.start();
        mSyncManager.start();
    }
}

Example 22 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class UfsVersionValidationTask method validateImpl.

@Override
protected ValidationTaskResult validateImpl(Map<String, String> optionMap) {
    UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(mConf).createMountSpecificConf(optionMap);
    String configuredVersion = mConf.getString(PropertyKey.UNDERFS_VERSION);
    List<String> availableVersions = UnderFileSystemFactoryRegistry.getSupportedVersions(mUfsPath, ufsConf);
    ValidationTaskResult result = new ValidationTaskResult();
    result.setName(getName());
    result.setDesc("Validates that the configured UFS version exists as a library on the " + "system.");
    if (!mConf.isSetByUser(PropertyKey.UNDERFS_VERSION)) {
        result.setState(ValidationUtils.State.SKIPPED);
        result.setOutput("The UFS version was not configured by the user.");
    } else if (availableVersions.contains(configuredVersion)) {
        result.setState(ValidationUtils.State.OK);
        result.setOutput(String.format("The UFS path %s with configured version %s is " + "supported by the current installation", mUfsPath, configuredVersion));
    } else {
        result.setState(ValidationUtils.State.FAILED);
        if (availableVersions.size() > 0) {
            result.setOutput(String.format("UFS path %s was configured with version %s. The " + "supported versions on this system are: %s", mUfsPath, configuredVersion, availableVersions.toString()));
        } else {
            result.setOutput(String.format("UFS path %s was configured with version %s. This " + "path does not support UFS version configuration.", mUfsPath, configuredVersion));
        }
        result.setAdvice(String.format("Configured UFS version %s not available. Check that " + "the version is correct. Otherwise, consider using a different version.", configuredVersion));
    }
    return result;
}

Example 23 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class UnderFileSystemContractTest method run.

/**
 * Runs the tests and returns nothing.
 */
public void run() throws Exception {
    UnderFileSystemConfiguration ufsConf = getUfsConf();
    UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find(mUfsPath, ufsConf);
    // Check if the ufs path is valid
    if (factory == null || !factory.supportsPath(mUfsPath)) {
        System.out.printf("%s is not a valid path", mUfsPath);
        System.exit(1);
    }
    // Set common properties
    mConf.set(PropertyKey.UNDERFS_LISTING_LENGTH, "50");
    mConf.set(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, "512B");
    // Increase the buffer time of journal writes to speed up tests
    mConf.set(PropertyKey.MASTER_JOURNAL_FLUSH_BATCH_TIME_MS, "1sec");
    mUfs = UnderFileSystem.Factory.create(mUfsPath, ufsConf);
    int failedCnt = runCommonOperations();
    if (mUfs.getUnderFSType().equals(S3_IDENTIFIER)) {
        failedCnt += runS3Operations();
    }
    System.out.printf("Tests completed with %d failed.%n", failedCnt);
}

Example 24 with UnderFileSystemConfiguration

use of alluxio.underfs.UnderFileSystemConfiguration in project alluxio by Alluxio.

the class UnderFileSystemContractTest method runValidationTask.

/**
 * Runs the tests and return a {@link ValidationTaskResult}.
 *
 * @return a task result for all UFS tests
 */
// TODO(jiacheng): Refactor and move this into ValidateHdfsMount
public ValidationTaskResult runValidationTask() throws IOException {
    Closer closer = Closer.create();
    final ByteArrayOutputStream msgBuf = new ByteArrayOutputStream();
    final ByteArrayOutputStream adviceBuf = new ByteArrayOutputStream();
    PrintStream msgStream = new PrintStream(msgBuf, true);
    PrintStream adviceStream = new PrintStream(adviceBuf, true);
    closer.register(msgStream);
    closer.register(adviceStream);
    closer.register(msgBuf);
    closer.register(adviceBuf);
    try {
        UnderFileSystemConfiguration ufsConf = getUfsConf();
        UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find(mUfsPath, ufsConf);
        // Check if the ufs path is valid
        if (factory == null || !factory.supportsPath(mUfsPath)) {
            msgStream.append(String.format("%s is not a valid path%n", mUfsPath));
            adviceStream.append(String.format("Please validate if %s is a correct path%n", mUfsPath));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, TASK_NAME, msgBuf.toString(), adviceBuf.toString());
        }
        // Set common properties
        mConf.set(PropertyKey.UNDERFS_LISTING_LENGTH, "50");
        mConf.set(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, "512B");
        // Increase the buffer time of journal writes to speed up tests
        mConf.set(PropertyKey.MASTER_JOURNAL_FLUSH_BATCH_TIME_MS, "1sec");
        mUfs = UnderFileSystem.Factory.create(mUfsPath, ufsConf);
        int failedCnt = runCommonOperations(msgStream, adviceStream, System.err);
        if (mUfs.getUnderFSType().equals(S3_IDENTIFIER)) {
            failedCnt += runS3Operations(msgStream, adviceStream, System.err);
        }
        msgStream.append(String.format("Tests completed with %d failed.%n", failedCnt));
        ValidationUtils.State state = failedCnt == 0 ? ValidationUtils.State.OK : ValidationUtils.State.FAILED;
        if (failedCnt > 0) {
            adviceStream.append("Please check the failed UFS operations from the output.");
        }
        return new ValidationTaskResult(state, TASK_NAME, msgBuf.toString(), adviceBuf.toString());
    } catch (Exception e) {
        msgStream.append(ValidationUtils.getErrorInfo(e));
        adviceStream.append("Please resolve the errors from failed UFS operations.");
        return new ValidationTaskResult(ValidationUtils.State.FAILED, TASK_NAME, msgBuf.toString(), adviceBuf.toString());
    } finally {
        closer.close();
    }
}