Examples with ListStatusPOptions alluxio.grpc.ListStatusPOptions used on opensource projects

Example 31 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class BaseFileSystem method loadMetadata.

@Override
public void loadMetadata(AlluxioURI path, final ListStatusPOptions options) throws FileDoesNotExistException, IOException, AlluxioException {
    checkUri(path);
    rpc(client -> {
        ListStatusPOptions mergedOptions = FileSystemOptions.listStatusDefaults(mFsContext.getPathConf(path)).toBuilder().mergeFrom(options).setLoadMetadataType(LoadMetadataPType.ALWAYS).setLoadMetadataOnly(true).build();
        client.listStatus(path, mergedOptions);
        return null;
    });
}

Example 32 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class ListStatusContext method mergeFrom.

/**
 * Merges and embeds the given {@link ListStatusPOptions} with the corresponding master options.
 *
 * @param optionsBuilder Builder for proto {@link ListStatusPOptions} to merge with defaults
 * @return the instance of {@link ListStatusContext} with default values for master
 */
public static ListStatusContext mergeFrom(ListStatusPOptions.Builder optionsBuilder) {
    ListStatusPOptions masterOptions = FileSystemOptions.listStatusDefaults(ServerConfiguration.global());
    ListStatusPOptions.Builder mergedOptionsBuilder = masterOptions.toBuilder().mergeFrom(optionsBuilder.build());
    return create(mergedOptionsBuilder);
}

Example 33 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class S3ClientRestApiTest method listBucketPagination.

@Test
public void listBucketPagination() throws Exception {
    AlluxioURI uri = new AlluxioURI("/bucket");
    mFileSystem.createDirectory(uri);
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder0"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file1"));
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    List<URIStatus> statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    // parameters with max-keys=1
    ListBucketResult expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1));
    String nextMarker = expected.getNextMarker();
    assertEquals("/bucket/file0", nextMarker);
    final Map<String, String> parameters = new HashMap<>();
    parameters.put("max-keys", "1");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("marker", nextMarker);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setMarker(nextMarker));
    nextMarker = expected.getNextMarker();
    assertEquals(1, expected.getContents().size());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("marker", nextMarker);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setMarker(nextMarker));
    nextMarker = expected.getNextMarker();
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with list-type=2 and max-key=1
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2));
    String nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/file0"), nextContinuationToken);
    parameters.remove("marker");
    parameters.put("list-type", "2");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("continuation-token", nextContinuationToken);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2).setContinuationToken(nextContinuationToken));
    nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/file1"), nextContinuationToken);
    assertEquals(1, expected.getContents().size());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("continuation-token", nextContinuationToken);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2).setContinuationToken(nextContinuationToken));
    nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/folder0"), nextContinuationToken);
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
}

Example 34 with ListStatusPOptions

use of alluxio.grpc.ListStatusPOptions in project alluxio by Alluxio.

the class S3ClientRestApiTest method listBucket.

@Test
public void listBucket() throws Exception {
    mFileSystem.createDirectory(new AlluxioURI("/bucket"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder0"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file1"));
    // empty parameters
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    List<URIStatus> statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    ListBucketResult expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", NO_PARAMS, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(6, expected.getContents().size());
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals("file1", expected.getContents().get(1).getKey());
    assertEquals("folder0/", expected.getContents().get(2).getKey());
    assertEquals("folder0/file0", expected.getContents().get(3).getKey());
    assertEquals("folder0/file1", expected.getContents().get(4).getKey());
    assertEquals("folder1/", expected.getContents().get(5).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with delimiter="/"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"));
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setDelimiter(AlluxioURI.SEPARATOR));
    Map<String, String> parameters = new HashMap<>();
    parameters.put("delimiter", AlluxioURI.SEPARATOR);
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(2, expected.getContents().size());
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals("file1", expected.getContents().get(1).getKey());
    assertEquals(2, expected.getCommonPrefixes().size());
    assertEquals("folder0/", expected.getCommonPrefixes().get(0).getPrefix());
    assertEquals("folder1/", expected.getCommonPrefixes().get(1).getPrefix());
    // parameters with prefix="folder0"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setPrefix("folder0"));
    parameters.clear();
    parameters.put("prefix", "folder0");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(3, expected.getContents().size());
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals("folder0/file0", expected.getContents().get(1).getKey());
    assertEquals("folder0/file1", expected.getContents().get(2).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with list-type=2 start-after="folder0/file0"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setListType(2).setStartAfter("file0"));
    parameters.clear();
    parameters.put("list-type", "2");
    parameters.put("start-after", "file0");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(5, expected.getContents().size());
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals("folder0/", expected.getContents().get(1).getKey());
    assertEquals("folder0/file0", expected.getContents().get(2).getKey());
    assertEquals("folder0/file1", expected.getContents().get(3).getKey());
    assertEquals("folder1/", expected.getContents().get(4).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
}