Examples with ListStatusPOptions alluxio.grpc.ListStatusPOptions used on opensource projects

Example 16 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class S3RestServiceHandler method getBucket.

/**
 * @summary gets a bucket and lists all the objects in it
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @param markerParam the optional marker param
 * @param prefixParam the optional prefix param
 * @param delimiterParam the optional delimiter param
 * @param encodingTypeParam optional encoding type param
 * @param maxKeysParam the optional max keys param
 * @param listTypeParam if listObjectV2 request
 * @param continuationTokenParam the optional continuationToken param for listObjectV2
 * @param startAfterParam  the optional startAfter param for listObjectV2
 * @return the response object
 */
@GET
@Path(BUCKET_PARAM)
public Response getBucket(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket, @QueryParam("marker") final String markerParam, @QueryParam("prefix") final String prefixParam, @QueryParam("delimiter") final String delimiterParam, @QueryParam("encoding-type") final String encodingTypeParam, @QueryParam("max-keys") final int maxKeysParam, @QueryParam("list-type") final int listTypeParam, @QueryParam("continuation-token") final String continuationTokenParam, @QueryParam("start-after") final String startAfterParam) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        String marker = markerParam == null ? "" : markerParam;
        String prefix = prefixParam == null ? "" : prefixParam;
        String encodingType = encodingTypeParam == null ? "url" : encodingTypeParam;
        int maxKeys = maxKeysParam <= 0 ? ListBucketOptions.DEFAULT_MAX_KEYS : maxKeysParam;
        String continuationToken = continuationTokenParam == null ? "" : continuationTokenParam;
        String startAfter = startAfterParam == null ? "" : startAfterParam;
        ListBucketOptions listBucketOptions = ListBucketOptions.defaults().setMarker(marker).setPrefix(prefix).setMaxKeys(maxKeys).setDelimiter(delimiterParam).setEncodingType(encodingType).setListType(listTypeParam).setContinuationToken(continuationToken).setStartAfter(startAfter);
        String path = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        List<URIStatus> children;
        try {
            // only list the direct children if delimiter is not null
            if (delimiterParam != null) {
                path = parsePath(path, prefix, delimiterParam);
                children = fs.listStatus(new AlluxioURI(path));
            } else {
                ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
                children = fs.listStatus(new AlluxioURI(path), options);
            }
        } catch (FileDoesNotExistException e) {
            // returned which does not match the S3 response behavior
            throw new S3Exception(e, bucket, S3ErrorCode.NO_SUCH_BUCKET);
        } catch (IOException | AlluxioException e) {
            throw new RuntimeException(e);
        }
        return new ListBucketResult(bucket, children, listBucketOptions);
    });
}

Example 17 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method mountPoint.

@Test
public void mountPoint() throws Exception {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_NEVER).build();
    List<String> rootListing = mFileSystem.listStatus(new AlluxioURI("/"), options).stream().map(URIStatus::getName).collect(Collectors.toList());
    assertEquals(1, rootListing.size());
    assertEquals("mnt", rootListing.get(0));
    options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_ALWAYS).build();
    rootListing = mFileSystem.listStatus(new AlluxioURI("/"), options).stream().map(URIStatus::getName).collect(Collectors.toList());
    assertEquals(1, rootListing.size());
    assertEquals("mnt", rootListing.get(0));
}

Example 18 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class UfsSyncIntegrationTest method mountPointNested.

@Test
public void mountPointNested() throws Exception {
    String ufsPath = Files.createTempDir().getAbsolutePath();
    mFileSystem.createDirectory(new AlluxioURI("/nested/mnt/"), CreateDirectoryPOptions.newBuilder().setRecursive(true).setWriteType(WritePType.CACHE_THROUGH).build());
    mFileSystem.mount(new AlluxioURI("/nested/mnt/ufs"), new AlluxioURI(ufsPath));
    // recursively sync (setAttribute enables recursive sync)
    mFileSystem.setAttribute(new AlluxioURI("/"), SetAttributePOptions.newBuilder().setCommonOptions(PSYNC_ALWAYS.toBuilder().setTtl(55555)).setRecursive(true).build());
    // Verify /nested/mnt/ dir has 1 mount point
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_NEVER).build();
    List<URIStatus> listing = mFileSystem.listStatus(new AlluxioURI("/nested/mnt/"), options);
    assertEquals(1, listing.size());
    assertEquals("ufs", listing.get(0).getName());
    // Remove a directory in the parent UFS, which has a mount point descendant
    URIStatus status = mFileSystem.getStatus(new AlluxioURI("/nested/mnt/"), GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_NEVER).build());
    assertTrue(new File(status.getUfsPath()).delete());
    // recursively sync (setAttribute enables recursive sync)
    mFileSystem.setAttribute(new AlluxioURI("/"), SetAttributePOptions.newBuilder().setCommonOptions(PSYNC_ALWAYS.toBuilder().setTtl(44444)).setRecursive(true).build());
    // Verify /nested/mnt/ dir has 1 mount point
    listing = mFileSystem.listStatus(new AlluxioURI("/nested/mnt/"), options);
    assertEquals(1, listing.size());
    assertEquals("ufs", listing.get(0).getName());
    // Remove a directory in the parent UFS, which has a mount point descendant
    status = mFileSystem.getStatus(new AlluxioURI("/nested/"), GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(PSYNC_NEVER).build());
    assertTrue(new File(status.getUfsPath()).delete());
    // recursively sync (setAttribute enables recursive sync)
    mFileSystem.setAttribute(new AlluxioURI("/"), SetAttributePOptions.newBuilder().setCommonOptions(PSYNC_ALWAYS.toBuilder().setTtl(44444)).setRecursive(true).build());
    // Verify /nested/mnt/ dir has 1 mount point
    listing = mFileSystem.listStatus(new AlluxioURI("/nested/mnt/"), options);
    assertEquals(1, listing.size());
    assertEquals("ufs", listing.get(0).getName());
    // adding a file into the nested mount point
    writeUfsFile(ufsPath + "/nestedufs", 1);
    // recursively sync (setAttribute enables recursive sync)
    mFileSystem.setAttribute(new AlluxioURI("/"), SetAttributePOptions.newBuilder().setCommonOptions(PSYNC_ALWAYS.toBuilder().setTtl(44444)).setRecursive(true).build());
    // Verify /nested/mnt/ufs dir has 1 file
    listing = mFileSystem.listStatus(new AlluxioURI("/nested/mnt/ufs"), options);
    assertEquals(1, listing.size());
    assertEquals("nestedufs", listing.get(0).getName());
}

Example 19 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method testNoTtlOnLoadedFiles.

@Test
public void testNoTtlOnLoadedFiles() throws Exception {
    int created = createUfsFiles(2);
    ServerConfiguration.set(PropertyKey.USER_FILE_METADATA_LOAD_TYPE, LoadMetadataType.ONCE.toString());
    ServerConfiguration.set(PropertyKey.USER_FILE_CREATE_TTL, "11000");
    ServerConfiguration.set(PropertyKey.USER_FILE_CREATE_TTL_ACTION, TtlAction.FREE.toString());
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(10000).setTtlAction(TtlAction.FREE).build()).build();
    List<URIStatus> list = mFileSystem.listStatus(new AlluxioURI("/mnt"), options);
    assertEquals(created + EXTRA_DIR_FILES, list.size());
    list.forEach(stat -> {
        assertEquals(-1, stat.getTtl());
    });
}

Example 20 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class LoadMetadataIntegrationTest method absentCacheMustCache.

@Test
public void absentCacheMustCache() throws Exception {
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).setLoadMetadataType(LoadMetadataPType.ONCE).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1)).build();
    GetStatusPOptions getStatusOptions = GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ONCE).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1)).build();
    // Each of these calls should only check one level of the directory,
    // because all the sub directories should be in the absent cache.
    checkListStatus("/mnt/mustcache/dir1/dir2", options, true, false, 1);
    checkListStatus("/mnt/mustcache/dir1", options, true, false, 1);
    checkListStatus("/mnt/mustcache", options, true, false, 1);
}