Examples with FileSystemContext alluxio.client.file.FileSystemContext used on opensource projects

Example 6 with FileSystemContext

use of alluxio.client.file.FileSystemContext in project alluxio by Alluxio.

the class Performance method main.

/**
 * Runs the performance test.
 *
 * Usage:
 * {@code java -cp <ALLUXIO-VERSION> alluxio.examples.Performance <MasterIp> <FileNamePrefix>
 * <WriteBlockSizeInBytes> <BlocksPerFile> <DebugMode:true/false> <Threads> <FilesPerThread>
 * <TestCaseNumber> <BaseFileNumber>}
 *
 * @param args the arguments for this example
 */
public static void main(String[] args) throws Exception {
    if (args.length < 9) {
        System.out.println("java -cp " + RuntimeConstants.ALLUXIO_JAR + " alluxio.examples.Performance " + "<MasterIp> <FileNamePrefix> <WriteBlockSizeInBytes> <BlocksPerFile> " + "<DebugMode:true/false> <Threads> <FilesPerThread> <TestCaseNumber> " + "<BaseFileNumber> [folderDir] \n" + "1: Files Write Test\n" + "2: Files Read Test\n" + "3: RamFile Write Test \n" + "4: RamFile Read Test \n" + "5: ByteBuffer Write Test \n" + "6: ByteBuffer Read Test \n");
        System.exit(-1);
    }
    InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    HostAndPort masterAddress = HostAndPort.fromString(args[0]);
    sFileName = args[1];
    sBlockSizeBytes = Integer.parseInt(args[2]);
    sBlocksPerFile = Long.parseLong(args[3]);
    sDebugMode = ("true".equals(args[4]));
    sThreads = Integer.parseInt(args[5]);
    sFiles = Integer.parseInt(args[6]) * sThreads;
    final int testCase = Integer.parseInt(args[7]);
    sBaseFileNumber = Integer.parseInt(args[8]);
    if (args.length > 9) {
        sFolder = args[9];
    }
    sFileBytes = sBlocksPerFile * sBlockSizeBytes;
    sFilesBytes = sFileBytes * sFiles;
    long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
    sResultPrefix = String.format("Threads %d FilesPerThread %d TotalFiles %d " + "BLOCK_SIZE_KB %d BLOCKS_PER_FILE %d FILE_SIZE_MB %d " + "Alluxio_WRITE_BUFFER_SIZE_KB %d BaseFileNumber %d : ", sThreads, sFiles / sThreads, sFiles, sBlockSizeBytes / 1024, sBlocksPerFile, sFileBytes / Constants.MB, fileBufferBytes / 1024, sBaseFileNumber);
    CommonUtils.warmUpLoop();
    conf.set(PropertyKey.MASTER_HOSTNAME, masterAddress.getHost());
    conf.set(PropertyKey.MASTER_RPC_PORT, Integer.toString(masterAddress.getPort()));
    FileSystemContext fsContext = FileSystemContext.create(conf);
    if (testCase == 1) {
        sResultPrefix = "AlluxioFilesWriteTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        AlluxioTest(true, /* write */
        FileSystem.Factory.create(fsContext));
    } else if (testCase == 2 || testCase == 9) {
        sResultPrefix = "AlluxioFilesReadTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        sAlluxioStreamingRead = (9 == testCase);
        AlluxioTest(false, /* read */
        FileSystem.Factory.create(fsContext));
    } else if (testCase == 3) {
        sResultPrefix = "RamFile Write " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(true, false);
    } else if (testCase == 4) {
        sResultPrefix = "RamFile Read " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(false, false);
    } else if (testCase == 5) {
        sResultPrefix = "ByteBuffer Write Test " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(true, true);
    } else if (testCase == 6) {
        sResultPrefix = "ByteBuffer Read Test " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(false, true);
    } else if (testCase == 7) {
        sResultPrefix = "HdfsFilesWriteTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        HdfsTest(true);
    } else if (testCase == 8) {
        sResultPrefix = "HdfsFilesReadTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        HdfsTest(false);
    } else {
        throw new RuntimeException("No Test Case " + testCase);
    }
    for (int k = 0; k < RESULT_ARRAY_SIZE; k++) {
        System.out.print(sResults[k] + " ");
    }
    System.out.println();
    System.exit(0);
}

Example 7 with FileSystemContext

use of alluxio.client.file.FileSystemContext in project alluxio by Alluxio.

the class JobUtils method loadThroughRead.

private static void loadThroughRead(URIStatus status, FileSystemContext context, long blockId, AlluxioConfiguration conf) throws IOException {
    // This does not work for remote worker unless we have passive cache on.
    AlluxioProperties prop = context.getClusterConf().copyProperties();
    prop.set(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED, true);
    AlluxioConfiguration config = new InstancedConfiguration(prop);
    FileSystemContext loadContext = FileSystemContext.create(config);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(loadContext);
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo info = Preconditions.checkNotNull(status.getBlockInfo(blockId));
    try (InputStream inputStream = blockStore.getInStream(info, inOptions, ImmutableMap.of())) {
        while (inputStream.read(sIgnoredReadBuf) != -1) {
        }
    }
}

Example 8 with FileSystemContext

use of alluxio.client.file.FileSystemContext in project alluxio by Alluxio.

the class JobUtils method loadBlock.

/**
 * Loads a block into the local worker. If the block doesn't exist in Alluxio, it will be read
 * from the UFS.
 * @param status the uriStatus
 * @param context filesystem context
 * @param blockId the id of the block to load
 * @param address specify a worker to load into
 * @param directCache Use passive-cache or direct cache request
 */
public static void loadBlock(URIStatus status, FileSystemContext context, long blockId, WorkerNetAddress address, boolean directCache) throws AlluxioException, IOException {
    AlluxioConfiguration conf = ServerConfiguration.global();
    // Explicitly specified a worker to load
    WorkerNetAddress localNetAddress = address;
    String localHostName = NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, conf);
    List<WorkerNetAddress> netAddress = context.getCachedWorkers().stream().map(BlockWorkerInfo::getNetAddress).filter(x -> Objects.equals(x.getHost(), localHostName)).collect(Collectors.toList());
    if (localNetAddress == null && !netAddress.isEmpty()) {
        localNetAddress = netAddress.get(0);
    }
    if (localNetAddress == null) {
        throw new NotFoundException(ExceptionMessage.NO_LOCAL_BLOCK_WORKER_LOAD_TASK.getMessage(blockId));
    }
    Set<String> pinnedLocation = status.getPinnedMediumTypes();
    if (pinnedLocation.size() > 1) {
        throw new AlluxioException(ExceptionMessage.PINNED_TO_MULTIPLE_MEDIUMTYPES.getMessage(status.getPath()));
    }
    // Only use this read local first method to load if nearest worker is clear
    if (netAddress.size() <= 1 && pinnedLocation.isEmpty() && status.isPersisted()) {
        if (directCache) {
            loadThroughCacheRequest(status, context, blockId, conf, localNetAddress);
        } else {
            loadThroughRead(status, context, blockId, conf);
        }
        return;
    }
    // TODO(bin): remove the following case when we consolidate tier and medium
    // since there is only one element in the set, we take the first element in the set
    String medium = pinnedLocation.isEmpty() ? "" : pinnedLocation.iterator().next();
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    // Set read location policy always to local first for loading blocks for job tasks
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    OutStreamOptions outOptions = OutStreamOptions.defaults(context.getClientContext());
    outOptions.setMediumType(medium);
    // Set write location policy always to local first for loading blocks for job tasks
    outOptions.setLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo blockInfo = status.getBlockInfo(blockId);
    Preconditions.checkNotNull(blockInfo, "Can not find block %s in status %s", blockId, status);
    long blockSize = blockInfo.getLength();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context);
    try (OutputStream outputStream = blockStore.getOutStream(blockId, blockSize, localNetAddress, outOptions)) {
        try (InputStream inputStream = blockStore.getInStream(blockId, inOptions)) {
            ByteStreams.copy(inputStream, outputStream);
        } catch (Throwable t) {
            try {
                ((Cancelable) outputStream).cancel();
            } catch (Throwable t2) {
                t.addSuppressed(t2);
            }
            throw t;
        }
    }
}

Example 9 with FileSystemContext

use of alluxio.client.file.FileSystemContext in project alluxio by Alluxio.

the class BasicCheckpoint method main.

/**
 * Example program for using checkpoints.
 * Usage: {@code java -cp <ALLUXIO-VERSION> alluxio.cli.BasicCheckpoint <FileFolder> <Files>}
 *
 * @param args the folder for the files and the files to use
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("java -cp " + RuntimeConstants.ALLUXIO_JAR + " alluxio.cli.BasicCheckpoint <FileFolder> <Files>");
        System.exit(-1);
    }
    FileSystemContext fsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()));
    boolean result = RunTestUtils.runExample(new BasicCheckpoint(args[0], Integer.parseInt(args[1]), fsContext));
    System.exit(result ? 0 : 1);
}

Example 10 with FileSystemContext

use of alluxio.client.file.FileSystemContext in project alluxio by Alluxio.

the class LogLevel method getTargetInfos.

private static List<TargetInfo> getTargetInfos(String[] targets, AlluxioConfiguration conf) throws IOException {
    // Trim the elements
    Set<String> targetSet = Arrays.stream(targets).map(String::trim).collect(Collectors.toSet());
    List<TargetInfo> targetInfoList = new ArrayList<>();
    // Allow plural form for the master/job_master and print a notice
    if (targetSet.contains(ROLE_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_MASTERS);
        targetSet.add(ROLE_MASTER);
        System.out.println("Target `masters` is replaced with `master`.");
    }
    if (targetSet.contains(ROLE_JOB_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary job master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_JOB_MASTERS);
        targetSet.add(ROLE_JOB_MASTER);
        System.out.println("Target `job_masters` is replaced with `job_master`.");
    }
    ClientContext clientContext = ClientContext.create(conf);
    // Created only when needed by master and workers
    FileSystemContext fsContext = null;
    // Created only when needed by the job master and job workers
    JobMasterClient jobClient = null;
    // Process each target
    for (String target : targetSet) {
        if (target.isEmpty()) {
            continue;
        } else if (target.equals(ROLE_MASTER)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(clientContext);
            }
            String masterHost = fsContext.getMasterAddress().getHostName();
            int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB, conf);
            TargetInfo master = new TargetInfo(masterHost, masterPort, ROLE_MASTER);
            targetInfoList.add(master);
        } else if (target.equals(ROLE_JOB_MASTER)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            String jobMasterHost = jobClient.getAddress().getHostName();
            int jobMasterPort = NetworkAddressUtils.getPort(ServiceType.JOB_MASTER_WEB, conf);
            TargetInfo jobMaster = new TargetInfo(jobMasterHost, jobMasterPort, ROLE_JOB_MASTER);
            targetInfoList.add(jobMaster);
        } else if (target.equals(ROLE_WORKERS)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(ClientContext.create(conf));
            }
            List<BlockWorkerInfo> workerInfoList = fsContext.getCachedWorkers();
            if (workerInfoList.size() == 0) {
                System.out.println("No workers found");
                System.exit(1);
            }
            for (BlockWorkerInfo workerInfo : workerInfoList) {
                WorkerNetAddress netAddress = workerInfo.getNetAddress();
                TargetInfo worker = new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), ROLE_WORKER);
                targetInfoList.add(worker);
            }
        } else if (target.equals(ROLE_JOB_WORKERS)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            List<JobWorkerHealth> jobWorkerInfoList = jobClient.getAllWorkerHealth();
            if (jobWorkerInfoList.size() == 0) {
                System.out.println("No job workers found");
                System.exit(1);
            }
            int jobWorkerPort = conf.getInt(PropertyKey.JOB_WORKER_WEB_PORT);
            for (JobWorkerHealth jobWorkerInfo : jobWorkerInfoList) {
                String jobWorkerHost = jobWorkerInfo.getHostname();
                TargetInfo jobWorker = new TargetInfo(jobWorkerHost, jobWorkerPort, ROLE_JOB_WORKER);
                targetInfoList.add(jobWorker);
            }
        } else if (target.contains(":")) {
            String[] hostPortPair = target.split(":");
            int port = Integer.parseInt(hostPortPair[1]);
            String role = inferRoleFromPort(port, conf);
            LOG.debug("Port {} maps to role {}", port, role);
            TargetInfo unspecifiedTarget = new TargetInfo(hostPortPair[0], port, role);
            System.out.format("Role inferred from port: %s%n", unspecifiedTarget);
            targetInfoList.add(unspecifiedTarget);
        } else {
            throw new IOException(String.format("Unrecognized target argument: %s. " + "Please pass the targets in the form of <host>:<port>, " + "with comma as the separator.", target));
        }
    }
    return targetInfoList;
}