Examples with AlluxioConfiguration alluxio.conf.AlluxioConfiguration used on opensource projects

Example 41 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class Benchmark method runSingleTask.

protected String runSingleTask(String[] args) throws Exception {
    // prepare the benchmark.
    prepare();
    if (!mBaseParameters.mProfileAgent.isEmpty()) {
        mBaseParameters.mJavaOpts.add("-javaagent:" + mBaseParameters.mProfileAgent + "=" + BaseParameters.AGENT_OUTPUT_PATH);
    }
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    String className = this.getClass().getCanonicalName();
    if (mBaseParameters.mCluster) {
        // run on job service
        long jobId = JobGrpcClientUtils.run(generateJobConfig(args), 0, conf);
        JobInfo jobInfo = JobGrpcClientUtils.getJobStatus(jobId, conf, true);
        return jobInfo.getResult().toString();
    }
    // run locally
    if (mBaseParameters.mInProcess) {
        // run in process
        T result = runLocal();
        if (mBaseParameters.mDistributed) {
            return result.toJson();
        }
        // aggregate the results
        final String s = result.aggregator().aggregate(Collections.singletonList(result)).toJson();
        return s;
    } else {
        // Spawn a new process
        List<String> command = new ArrayList<>();
        command.add(conf.get(PropertyKey.HOME) + "/bin/alluxio");
        command.add("runClass");
        command.add(className);
        command.addAll(Arrays.asList(args));
        command.add(BaseParameters.IN_PROCESS_FLAG);
        command.addAll(mBaseParameters.mJavaOpts.stream().map(String::trim).collect(Collectors.toList()));
        LOG.info("running command: " + String.join(" ", command));
        return ShellUtils.execCommand(command.toArray(new String[0]));
    }
}

Example 42 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class FileSystemMasterClientPoolTest method create.

@Test
public void create() throws Exception {
    AlluxioConfiguration conf = ConfigurationTestUtils.defaults();
    FileSystemMasterClient expectedClient = Mockito.mock(FileSystemMasterClient.class);
    PowerMockito.mockStatic(FileSystemMasterClient.Factory.class);
    Mockito.when(FileSystemMasterClient.Factory.create(Mockito.any(MasterClientContext.class))).thenReturn(expectedClient);
    FileSystemMasterClient client;
    ClientContext clientContext = ClientContext.create(conf);
    MasterInquireClient masterInquireClient = MasterInquireClient.Factory.create(conf, clientContext.getUserState());
    MasterClientContext masterClientContext = MasterClientContext.newBuilder(clientContext).setMasterInquireClient(masterInquireClient).build();
    try (FileSystemMasterClientPool pool = new FileSystemMasterClientPool(masterClientContext)) {
        client = pool.acquire();
        assertEquals(expectedClient, client);
        pool.release(client);
    }
    Mockito.verify(client).close();
}

Example 43 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class FuseManager method start.

/**
 * Starts mounting the internal Fuse applications.
 */
public void start() {
    AlluxioConfiguration conf = ServerConfiguration.global();
    if (!conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_POINT) || conf.getString(PropertyKey.WORKER_FUSE_MOUNT_POINT).isEmpty()) {
        LOG.error("Failed to launch worker internal Fuse application. {} should be set.", PropertyKey.WORKER_FUSE_MOUNT_POINT);
        return;
    }
    if (!conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH) || conf.getString(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH).isEmpty()) {
        LOG.error("Failed to launch worker internal Fuse application. {} should be set.", PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH.getName());
        return;
    }
    String fuseMount = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_POINT);
    String alluxioPath = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH);
    // create the folder if it does not exist
    try {
        String[] fuseOptsSeparated = new String[0];
        if (conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_OPTIONS)) {
            String fuseOptsString = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_OPTIONS);
            if (!fuseOptsString.isEmpty()) {
                fuseOptsSeparated = fuseOptsString.split(FUSE_OPTION_SEPARATOR);
            }
        }
        List<String> fuseOptions = AlluxioFuse.parseFuseOptions(fuseOptsSeparated, conf);
        FuseMountOptions options = new FuseMountOptions(fuseMount, alluxioPath, conf.getBoolean(PropertyKey.FUSE_DEBUG_ENABLED), fuseOptions);
        // TODO(lu) consider launching fuse in a separate thread as blocking operation
        // so that we can know about the fuse application status
        FileSystem fileSystem = mResourceCloser.register(FileSystem.Factory.create(mFsContext));
        mFuseUmountable = AlluxioFuse.launchFuse(fileSystem, conf, options, false);
    } catch (Throwable throwable) {
        // TODO(lu) for already mounted application, unmount first and then remount
        LOG.error("Failed to launch worker internal Fuse application", throwable);
    }
}

Example 44 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class LoadCommand method runLoadTask.

private void runLoadTask(AlluxioURI filePath, URIStatus status, boolean local) throws IOException {
    AlluxioConfiguration conf = mFsContext.getPathConf(filePath);
    OpenFilePOptions options = FileSystemOptions.openFileDefaults(conf);
    BlockLocationPolicy policy = Preconditions.checkNotNull(BlockLocationPolicy.Factory.create(conf.getString(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), conf), "UFS read location policy Required when loading files");
    WorkerNetAddress dataSource;
    List<Long> blockIds = status.getBlockIds();
    for (long blockId : blockIds) {
        if (local) {
            dataSource = mFsContext.getNodeLocalWorker();
        } else {
            // send request to data source
            AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext);
            Pair<WorkerNetAddress, BlockInStream.BlockInStreamSource> dataSourceAndType = blockStore.getDataSourceAndType(status.getBlockInfo(blockId), status, policy, ImmutableMap.of());
            dataSource = dataSourceAndType.getFirst();
        }
        Protocol.OpenUfsBlockOptions openUfsBlockOptions = new InStreamOptions(status, options, conf).getOpenUfsBlockOptions(blockId);
        cacheBlock(blockId, dataSource, status, openUfsBlockOptions);
    }
}

Example 45 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class DistributedCpCommand method run.

@Override
public int run(CommandLine cl) throws AlluxioException, IOException {
    mActiveJobs = FileSystemShellUtils.getIntArg(cl, ACTIVE_JOB_COUNT_OPTION, AbstractDistributedJobCommand.DEFAULT_ACTIVE_JOBS);
    System.out.format("Allow up to %s active jobs%n", mActiveJobs);
    boolean overwrite = FileSystemShellUtils.getBoolArg(cl, OVERWRITE_OPTION, true);
    String[] args = cl.getArgs();
    AlluxioURI srcPath = new AlluxioURI(args[0]);
    AlluxioURI dstPath = new AlluxioURI(args[1]);
    if (PathUtils.hasPrefix(dstPath.toString(), srcPath.toString())) {
        throw new RuntimeException(ExceptionMessage.MIGRATE_CANNOT_BE_TO_SUBDIRECTORY.getMessage(srcPath, dstPath));
    }
    AlluxioConfiguration conf = mFsContext.getPathConf(dstPath);
    mWriteType = conf.getString(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT);
    int defaultBatchSize = conf.getInt(PropertyKey.JOB_REQUEST_BATCH_SIZE);
    int batchSize = FileSystemShellUtils.getIntArg(cl, BATCH_SIZE_OPTION, defaultBatchSize);
    distributedCp(srcPath, dstPath, overwrite, batchSize);
    return 0;
}