Examples with ClientContext alluxio.ClientContext used on opensource projects

Example 6 with ClientContext

use of alluxio.ClientContext in project alluxio by Alluxio.

the class AlluxioFuseFileSystem method statfsInternal.

private int statfsInternal(String path, Statvfs stbuf) {
    ClientContext ctx = ClientContext.create(sConf);
    try (BlockMasterClient blockClient = BlockMasterClient.Factory.create(MasterClientContext.newBuilder(ctx).build())) {
        Set<BlockMasterInfo.BlockMasterInfoField> blockMasterInfoFilter = new HashSet<>(Arrays.asList(BlockMasterInfo.BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfo.BlockMasterInfoField.FREE_BYTES, BlockMasterInfo.BlockMasterInfoField.USED_BYTES));
        BlockMasterInfo blockMasterInfo = blockClient.getBlockMasterInfo(blockMasterInfoFilter);
        // although user may set different block size for different files,
        // small block size can result more accurate compute.
        long blockSize = 4L * Constants.KB;
        // fs block size
        // The size in bytes of the minimum unit of allocation on this file system
        stbuf.f_bsize.set(blockSize);
        // The preferred length of I/O requests for files on this file system.
        stbuf.f_frsize.set(blockSize);
        // total data blocks in fs
        stbuf.f_blocks.set(blockMasterInfo.getCapacityBytes() / blockSize);
        // free blocks in fs
        long freeBlocks = blockMasterInfo.getFreeBytes() / blockSize;
        stbuf.f_bfree.set(freeBlocks);
        stbuf.f_bavail.set(freeBlocks);
        // inode info in fs
        // TODO(liuhongtong): support inode info
        stbuf.f_files.set(UNKNOWN_INODES);
        stbuf.f_ffree.set(UNKNOWN_INODES);
        stbuf.f_favail.set(UNKNOWN_INODES);
        // max file name length
        stbuf.f_namemax.set(MAX_NAME_LENGTH);
    } catch (IOException e) {
        LOG.error("statfs({}) failed:", path, e);
        return -ErrorCodes.EIO();
    }
    return 0;
}

Example 7 with ClientContext

use of alluxio.ClientContext in project alluxio by Alluxio.

the class AlluxioJniFuseFileSystem method statfsInternal.

private int statfsInternal(String path, Statvfs stbuf) {
    ClientContext ctx = ClientContext.create(mConf);
    try (BlockMasterClient blockClient = BlockMasterClient.Factory.create(MasterClientContext.newBuilder(ctx).build())) {
        Set<BlockMasterInfo.BlockMasterInfoField> blockMasterInfoFilter = new HashSet<>(Arrays.asList(BlockMasterInfo.BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfo.BlockMasterInfoField.FREE_BYTES, BlockMasterInfo.BlockMasterInfoField.USED_BYTES));
        BlockMasterInfo blockMasterInfo = blockClient.getBlockMasterInfo(blockMasterInfoFilter);
        // although user may set different block size for different files,
        // small block size can result more accurate compute.
        long blockSize = 4L * Constants.KB;
        // fs block size
        // The size in bytes of the minimum unit of allocation on this file system
        stbuf.f_bsize.set(blockSize);
        // The preferred length of I/O requests for files on this file system.
        stbuf.f_frsize.set(blockSize);
        // total data blocks in fs
        stbuf.f_blocks.set(blockMasterInfo.getCapacityBytes() / blockSize);
        // free blocks in fs
        long freeBlocks = blockMasterInfo.getFreeBytes() / blockSize;
        stbuf.f_bfree.set(freeBlocks);
        stbuf.f_bavail.set(freeBlocks);
        // inode info in fs
        stbuf.f_files.set(UNKNOWN_INODES);
        stbuf.f_ffree.set(UNKNOWN_INODES);
        stbuf.f_favail.set(UNKNOWN_INODES);
        // max file name length
        stbuf.f_namemax.set(MAX_NAME_LENGTH);
    } catch (IOException e) {
        LOG.error("statfs({}) failed:", path, e);
        return -ErrorCodes.EIO();
    }
    return 0;
}

Example 8 with ClientContext

use of alluxio.ClientContext in project alluxio by Alluxio.

the class LogLevel method getTargetInfos.

private static List<TargetInfo> getTargetInfos(String[] targets, AlluxioConfiguration conf) throws IOException {
    // Trim the elements
    Set<String> targetSet = Arrays.stream(targets).map(String::trim).collect(Collectors.toSet());
    List<TargetInfo> targetInfoList = new ArrayList<>();
    // Allow plural form for the master/job_master and print a notice
    if (targetSet.contains(ROLE_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_MASTERS);
        targetSet.add(ROLE_MASTER);
        System.out.println("Target `masters` is replaced with `master`.");
    }
    if (targetSet.contains(ROLE_JOB_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary job master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_JOB_MASTERS);
        targetSet.add(ROLE_JOB_MASTER);
        System.out.println("Target `job_masters` is replaced with `job_master`.");
    }
    ClientContext clientContext = ClientContext.create(conf);
    // Created only when needed by master and workers
    FileSystemContext fsContext = null;
    // Created only when needed by the job master and job workers
    JobMasterClient jobClient = null;
    // Process each target
    for (String target : targetSet) {
        if (target.isEmpty()) {
            continue;
        } else if (target.equals(ROLE_MASTER)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(clientContext);
            }
            String masterHost = fsContext.getMasterAddress().getHostName();
            int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB, conf);
            TargetInfo master = new TargetInfo(masterHost, masterPort, ROLE_MASTER);
            targetInfoList.add(master);
        } else if (target.equals(ROLE_JOB_MASTER)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            String jobMasterHost = jobClient.getAddress().getHostName();
            int jobMasterPort = NetworkAddressUtils.getPort(ServiceType.JOB_MASTER_WEB, conf);
            TargetInfo jobMaster = new TargetInfo(jobMasterHost, jobMasterPort, ROLE_JOB_MASTER);
            targetInfoList.add(jobMaster);
        } else if (target.equals(ROLE_WORKERS)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(ClientContext.create(conf));
            }
            List<BlockWorkerInfo> workerInfoList = fsContext.getCachedWorkers();
            if (workerInfoList.size() == 0) {
                System.out.println("No workers found");
                System.exit(1);
            }
            for (BlockWorkerInfo workerInfo : workerInfoList) {
                WorkerNetAddress netAddress = workerInfo.getNetAddress();
                TargetInfo worker = new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), ROLE_WORKER);
                targetInfoList.add(worker);
            }
        } else if (target.equals(ROLE_JOB_WORKERS)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            List<JobWorkerHealth> jobWorkerInfoList = jobClient.getAllWorkerHealth();
            if (jobWorkerInfoList.size() == 0) {
                System.out.println("No job workers found");
                System.exit(1);
            }
            int jobWorkerPort = conf.getInt(PropertyKey.JOB_WORKER_WEB_PORT);
            for (JobWorkerHealth jobWorkerInfo : jobWorkerInfoList) {
                String jobWorkerHost = jobWorkerInfo.getHostname();
                TargetInfo jobWorker = new TargetInfo(jobWorkerHost, jobWorkerPort, ROLE_JOB_WORKER);
                targetInfoList.add(jobWorker);
            }
        } else if (target.contains(":")) {
            String[] hostPortPair = target.split(":");
            int port = Integer.parseInt(hostPortPair[1]);
            String role = inferRoleFromPort(port, conf);
            LOG.debug("Port {} maps to role {}", port, role);
            TargetInfo unspecifiedTarget = new TargetInfo(hostPortPair[0], port, role);
            System.out.format("Role inferred from port: %s%n", unspecifiedTarget);
            targetInfoList.add(unspecifiedTarget);
        } else {
            throw new IOException(String.format("Unrecognized target argument: %s. " + "Please pass the targets in the form of <host>:<port>, " + "with comma as the separator.", target));
        }
    }
    return targetInfoList;
}

Example 9 with ClientContext

use of alluxio.ClientContext in project alluxio by Alluxio.

the class GetConfTest method getConfByAlias.

@Test
public void getConfByAlias() {
    PropertyKey testProperty = new PropertyKey.Builder("alluxio.test.property").setAlias(new String[] { "alluxio.test.property.alias" }).setDefaultValue("testValue").build();
    ClientContext ctx = ClientContext.create(ServerConfiguration.global());
    assertEquals(0, GetConf.getConf(ctx, "alluxio.test.property.alias"));
    assertEquals("testValue\n", mOutputStream.toString());
    mOutputStream.reset();
    assertEquals(0, GetConf.getConf(ctx, "alluxio.test.property"));
    assertEquals("testValue\n", mOutputStream.toString());
    PropertyKey.unregister(testProperty);
}

Example 10 with ClientContext

use of alluxio.ClientContext in project alluxio by Alluxio.

the class GetConfTest method getConfWithInvalidConf.

@Test
public void getConfWithInvalidConf() throws Exception {
    try (Closeable p = new SystemPropertyRule(ImmutableMap.of(PropertyKey.CONF_VALIDATION_ENABLED.toString(), "false", PropertyKey.ZOOKEEPER_ENABLED.toString(), "true")).toResource()) {
        ServerConfiguration.reset();
        ClientContext ctx = ClientContext.create(ServerConfiguration.global());
        assertEquals(0, GetConf.getConf(ctx, PropertyKey.ZOOKEEPER_ENABLED.toString()));
        assertEquals("true\n", mOutputStream.toString());
    } finally {
        ServerConfiguration.reset();
    }
}