Examples with ListStatusPOptions alluxio.grpc.ListStatusPOptions used on opensource projects

Example 11 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method listDirSync.

@Test
public void listDirSync() throws Exception {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_ALWAYS).build();
    checkListStatus(ROOT_DIR, options, true);
    // Create new ufs paths.
    new File(ufsPath(NEW_DIR)).mkdirs();
    writeUfsFile(ufsPath(NEW_FILE), 2);
    checkListStatus(ROOT_DIR, options, true);
}

Example 12 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method listDirSyncInterval.

@Test(timeout = 10000)
public void listDirSyncInterval() throws Exception {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_INTERVAL).build();
    long startMs = System.currentTimeMillis();
    List<URIStatus> statusList = mFileSystem.listStatus(new AlluxioURI(alluxioPath(ROOT_DIR)), options);
    long startSize = statusList.size();
    int index = 10;
    while (true) {
        writeUfsFile(ufsPath(NEW_FILE + index), 1);
        statusList = mFileSystem.listStatus(new AlluxioURI(alluxioPath(ROOT_DIR)), options);
        if (statusList.size() != startSize) {
            break;
        }
        index++;
    }
    long endMs = System.currentTimeMillis();
    assertTrue((endMs - startMs) >= INTERVAL_MS);
}

Example 13 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class DistributedLoadUtils method load.

/**
 * Loads a file or directory in Alluxio space, makes it resident in memory.
 *
 * @param command The command to execute loading
 * @param pool The pool for batched jobs
 * @param filePath The {@link AlluxioURI} path to load into Alluxio memory
 * @param replication Number of block replicas of each loaded file
 * @param workerSet A set of worker hosts to load data
 * @param excludedWorkerSet A set of worker hosts can not to load data
 * @param localityIds The locality identify set
 * @param excludedLocalityIds A set of worker locality identify can not to load data
 * @param printOut whether print out progress in console
 * @throws AlluxioException when Alluxio exception occurs
 * @throws IOException when non-Alluxio exception occurs
 */
private static void load(AbstractDistributedJobCommand command, List<URIStatus> pool, int batchSize, AlluxioURI filePath, int replication, Set<String> workerSet, Set<String> excludedWorkerSet, Set<String> localityIds, Set<String> excludedLocalityIds, boolean directCache, boolean printOut) throws IOException, AlluxioException {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    LongAdder incompleteCount = new LongAdder();
    command.mFileSystem.iterateStatus(filePath, options, uriStatus -> {
        if (!uriStatus.isFolder()) {
            if (!uriStatus.isCompleted()) {
                incompleteCount.increment();
                System.out.printf("Ignored load because: %s is in incomplete status", uriStatus.getPath());
                return;
            }
            AlluxioURI fileURI = new AlluxioURI(uriStatus.getPath());
            if (uriStatus.getInAlluxioPercentage() == 100 && replication == 1) {
                // The file has already been fully loaded into Alluxio.
                if (printOut) {
                    System.out.println(fileURI + " is already fully loaded in Alluxio");
                }
                return;
            }
            pool.add(uriStatus);
            if (pool.size() == batchSize) {
                addJob(command, pool, replication, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, directCache, printOut);
                pool.clear();
            }
        }
    });
    if (incompleteCount.longValue() > 0) {
        System.out.printf("Ignore load %d paths because they are in incomplete status", incompleteCount.longValue());
    }
}

Example 14 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class DuCommand method runPlainPath.

@Override
protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
    ListStatusPOptions listOptions = ListStatusPOptions.newBuilder().setRecursive(true).build();
    // whether to print info of human readable format
    boolean readable = cl.hasOption(READABLE_OPTION_NAME);
    // whether to group by worker
    boolean groupByWorker = cl.hasOption(GROUP_BY_WORKER_OPTION_NAME);
    // whether to display the memory size and percentage information
    boolean addMemory = cl.hasOption(MEMORY_OPTION_NAME);
    Optional<String> workerHostName = groupByWorker ? Optional.of("total") : Optional.empty();
    if (cl.hasOption(SUMMARIZE_OPTION_NAME)) {
        AtomicLong totalSize = new AtomicLong();
        AtomicLong sizeInAlluxio = new AtomicLong();
        AtomicLong sizeInMem = new AtomicLong();
        Map<String, Long> distributionMap = new HashMap<>();
        mFileSystem.iterateStatus(path, listOptions, status -> {
            if (!status.isFolder()) {
                long size = status.getLength();
                totalSize.addAndGet(size);
                sizeInMem.addAndGet(size * status.getInMemoryPercentage());
                sizeInAlluxio.addAndGet(size * status.getInAlluxioPercentage());
            }
            if (groupByWorker) {
                fillDistributionMap(distributionMap, status);
            }
        });
        String sizeMessage = readable ? FormatUtils.getSizeFromBytes(totalSize.get()) : String.valueOf(totalSize);
        String inAlluxioMessage = getFormattedValues(readable, sizeInAlluxio.get() / 100, totalSize.get());
        Optional<String> inMemMessage = addMemory ? Optional.of(getFormattedValues(readable, sizeInMem.get() / 100, totalSize.get())) : Optional.empty();
        printInfo(sizeMessage, inAlluxioMessage, path.toString(), inMemMessage, workerHostName);
        // If workerHostName and inMemMessage is present, the "In Memory" columns
        // need an empty string as placeholders.
        // Otherwise we use an empty Optional.
        // e.g. inMemMessage is present, inMem should be ""
        // File Size     In Alluxio       In Memory        Worker Host Name          Path
        // 2             2                2                total                     /
        // 2                                 node1
        // e.g. inMemMessage is not present, inMem should be an empty Optional
        // File Size     In Alluxio       Worker Host Name          Path
        // 2             2                total                     /
        // 2                node1
        Optional<String> inMem = inMemMessage.isPresent() ? Optional.of("") : inMemMessage;
        getSizeInfoGroupByWorker(distributionMap, readable, inMem);
    } else {
        List<URIStatus> statuses = mFileSystem.listStatus(path, listOptions);
        if (statuses == null || statuses.size() == 0) {
            return;
        }
        statuses.sort(Comparator.comparing(URIStatus::getPath));
        for (URIStatus status : statuses) {
            if (!status.isFolder()) {
                long totalSize = status.getLength();
                String sizeMessage = readable ? FormatUtils.getSizeFromBytes(totalSize) : String.valueOf(totalSize);
                String inAlluxioMessage = getFormattedValues(readable, status.getInAlluxioPercentage() * totalSize / 100, totalSize);
                Optional<String> inMemMessage = addMemory ? Optional.of(getFormattedValues(readable, status.getInMemoryPercentage() * totalSize / 100, totalSize)) : Optional.empty();
                Map<String, Long> distributionMap = new HashMap<>();
                if (groupByWorker) {
                    fillDistributionMap(distributionMap, status);
                }
                Optional<String> inMem = inMemMessage.isPresent() ? Optional.of("") : inMemMessage;
                printInfo(sizeMessage, inAlluxioMessage, status.getPath(), inMemMessage, workerHostName);
                getSizeInfoGroupByWorker(distributionMap, readable, inMem);
            }
        }
    }
}

Example 15 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class CountCommand method run.

@Override
public int run(CommandLine cl) throws AlluxioException, IOException {
    String[] args = cl.getArgs();
    AlluxioURI inputPath = new AlluxioURI(args[0]);
    AtomicLong fileCount = new AtomicLong();
    AtomicLong folderCount = new AtomicLong();
    AtomicLong folderSize = new AtomicLong();
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    mFileSystem.iterateStatus(inputPath, options, uriStatus -> {
        if (uriStatus.isFolder()) {
            folderCount.incrementAndGet();
        } else {
            fileCount.incrementAndGet();
            folderSize.getAndAdd(uriStatus.getLength());
        }
    });
    printInfo(cl.hasOption(READABLE_OPTION_NAME), fileCount.get(), folderCount.get(), folderSize.get());
    return 0;
}