Examples with ListStatusPOptions alluxio.grpc.ListStatusPOptions used on opensource projects

Example 6 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class ConcurrentFileSystemMasterCreateIntegrationTest method runLoadMetadata.

/**
 * Runs load metadata tests.
 *
 * @param writeType the {@link WritePType} to create ancestors, if not null
 * @param useSinglePath if true, threads will only use a single path
 * @param createFiles if true, will create files at the bottom of the tree, directories otherwise
 * @param listParentDir if true, will list the parent dir to load the metadata
 */
private void runLoadMetadata(WritePType writeType, boolean useSinglePath, boolean createFiles, boolean listParentDir) throws Exception {
    int numThreads = CONCURRENCY_FACTOR;
    int bufferFactor = CONCURRENCY_FACTOR / 2;
    // 2 nested components to create.
    long limitMs = 2 * SLEEP_MS * bufferFactor;
    int uniquePaths = useSinglePath ? 1 : numThreads;
    if (writeType != WritePType.CACHE_THROUGH) {
        // all 3 components must be synced to UFS.
        limitMs = 3 * SLEEP_MS * bufferFactor;
    }
    if (listParentDir) {
        // Loading direct children needs to load each child, so reduce the branching factor.
        uniquePaths = 10;
        limitMs = 2 + uniquePaths * SLEEP_MS * bufferFactor;
    }
    // Create UFS files outside of Alluxio.
    new File(mLocalUfsPath + "/existing/path/").mkdirs();
    for (int i = 0; i < uniquePaths; i++) {
        if (createFiles) {
            FileWriter fileWriter = new FileWriter(mLocalUfsPath + "/existing/path/last_" + i);
            fileWriter.write("testtesttesttest");
            fileWriter.close();
        } else {
            new File(mLocalUfsPath + "/existing/path/last_" + i).mkdirs();
        }
    }
    if (writeType != null) {
        // create inodes in Alluxio
        mFileSystem.createDirectory(new AlluxioURI("/existing/path/"), CreateDirectoryPOptions.newBuilder().setRecursive(true).setWriteType(writeType).build());
    }
    // Generate path names for threads.
    AlluxioURI[] paths = new AlluxioURI[numThreads];
    int fileId = 0;
    for (int i = 0; i < numThreads; i++) {
        if (listParentDir) {
            paths[i] = new AlluxioURI("/existing/path/");
        } else {
            paths[i] = new AlluxioURI("/existing/path/last_" + ((fileId++) % uniquePaths));
        }
    }
    List<Throwable> errors;
    if (listParentDir) {
        errors = ConcurrentFileSystemMasterUtils.unaryOperation(mFileSystem, ConcurrentFileSystemMasterUtils.UnaryOperation.LIST_STATUS, paths, limitMs);
    } else {
        errors = ConcurrentFileSystemMasterUtils.unaryOperation(mFileSystem, ConcurrentFileSystemMasterUtils.UnaryOperation.GET_FILE_INFO, paths, limitMs);
    }
    if (!errors.isEmpty()) {
        Assert.fail("Encountered " + errors.size() + " errors, the first one is " + errors.get(0) + "\n" + Throwables.getStackTraceAsString(errors.get(0)));
    }
    ListStatusPOptions listOptions = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).build();
    List<URIStatus> files = mFileSystem.listStatus(new AlluxioURI("/"), listOptions);
    Assert.assertEquals(1, files.size());
    Assert.assertEquals("existing", files.get(0).getName());
    Assert.assertEquals(PersistenceState.PERSISTED, PersistenceState.valueOf(files.get(0).getPersistenceState()));
    files = mFileSystem.listStatus(new AlluxioURI("/existing"), listOptions);
    Assert.assertEquals(1, files.size());
    Assert.assertEquals("path", files.get(0).getName());
    Assert.assertEquals(PersistenceState.PERSISTED, PersistenceState.valueOf(files.get(0).getPersistenceState()));
    files = mFileSystem.listStatus(new AlluxioURI("/existing/path/"), listOptions);
    Assert.assertEquals(uniquePaths, files.size());
    Collections.sort(files, new ConcurrentFileSystemMasterUtils.IntegerSuffixedPathComparator());
    for (int i = 0; i < uniquePaths; i++) {
        Assert.assertEquals("last_" + i, files.get(i).getName());
        Assert.assertEquals(PersistenceState.PERSISTED, PersistenceState.valueOf(files.get(i).getPersistenceState()));
    }
}

Example 7 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method loadRecursive.

@Test
public void loadRecursive() throws Exception {
    ServerConfiguration.set(PropertyKey.USER_FILE_METADATA_LOAD_TYPE, LoadMetadataType.ONCE.toString());
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    int createdInodes = createUfsFiles(5);
    List<URIStatus> list = mFileSystem.listStatus(new AlluxioURI("/mnt"), options);
    // 25 files, 25 level 2 dirs, 5 level 1 dirs, 1 file and 1 dir created in before
    // 4 directories/files in must cache
    assertEquals(createdInodes + EXTRA_DIR_FILES, list.size());
}

Example 8 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method symLink.

@Test
public void symLink() throws Exception {
    Path target = Paths.get(mLocalUfsPath + "/dir1/dirA/file");
    Path link = Paths.get(mLocalUfsPath + "/symlink");
    Files.createSymbolicLink(link, target);
    Path target2 = Paths.get(mLocalUfsPath + "/dir1/dirA/");
    Path link2 = Paths.get(mLocalUfsPath + "/symlinkDir");
    Files.createSymbolicLink(link2, target2);
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ONCE).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1)).build();
    List<URIStatus> statusList = checkListStatus("/mnt/", options, true, true, 3);
    assertEquals(4, statusList.size());
    assertTrue(statusList.stream().anyMatch(x -> x.getName().equals("symlink")));
    assertTrue(statusList.stream().anyMatch(x -> x.getName().equals("symlinkDir")));
    if (Files.exists(link)) {
        Files.delete(link);
    }
    if (Files.exists(link2)) {
        Files.delete(link2);
    }
}

Example 9 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method repeatedDirectorySync.

@Test
public void repeatedDirectorySync() throws Exception {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).setLoadMetadataType(LoadMetadataPType.ONCE).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1)).build();
    checkListStatus("/mnt/dir1/", options, true, true, 3);
    checkListStatus("/mnt/dir1/", options, true, true, 0);
}

Example 10 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method getStatusDoesNotLoadFile.

@Test
public void getStatusDoesNotLoadFile() throws Exception {
    GetStatusPOptions getStatusOptions = GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ONCE).build();
    ListStatusPOptions lsOptions = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ALWAYS).setCommonOptions(PSYNC_ALWAYS).build();
    // load metadata for dirA with 'ALWAYS'
    checkGetStatus("/mnt/dir1/dirA", getStatusOptions, true, true, 2);
    // the file should NOT already be loaded
    checkGetStatus("/mnt/dir1/dirA/file", getStatusOptions, true, true, 1);
}