Examples with ListBucketResult alluxio.proxy.s3.ListBucketResult used on opensource projects

Example 1 with ListBucketResult

use of alluxio.proxy.s3.ListBucketResult in project alluxio by Alluxio.

the class S3ClientRestApiTest method listBucketPagination.

@Test
public void listBucketPagination() throws Exception {
    AlluxioURI uri = new AlluxioURI("/bucket");
    mFileSystem.createDirectory(uri);
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder0"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file1"));
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    List<URIStatus> statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    // parameters with max-keys=1
    ListBucketResult expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1));
    String nextMarker = expected.getNextMarker();
    assertEquals("/bucket/file0", nextMarker);
    final Map<String, String> parameters = new HashMap<>();
    parameters.put("max-keys", "1");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("marker", nextMarker);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setMarker(nextMarker));
    nextMarker = expected.getNextMarker();
    assertEquals(1, expected.getContents().size());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("marker", nextMarker);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setMarker(nextMarker));
    nextMarker = expected.getNextMarker();
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with list-type=2 and max-key=1
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2));
    String nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/file0"), nextContinuationToken);
    parameters.remove("marker");
    parameters.put("list-type", "2");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("continuation-token", nextContinuationToken);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2).setContinuationToken(nextContinuationToken));
    nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/file1"), nextContinuationToken);
    assertEquals(1, expected.getContents().size());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    parameters.put("continuation-token", nextContinuationToken);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setMaxKeys(1).setListType(2).setContinuationToken(nextContinuationToken));
    nextContinuationToken = expected.getNextContinuationToken();
    assertEquals(ListBucketResult.encodeToken("/bucket/folder0"), nextContinuationToken);
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
}

Example 2 with ListBucketResult

use of alluxio.proxy.s3.ListBucketResult in project alluxio by Alluxio.

the class S3ClientRestApiTest method listBucket.

@Test
public void listBucket() throws Exception {
    mFileSystem.createDirectory(new AlluxioURI("/bucket"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder0"));
    mFileSystem.createDirectory(new AlluxioURI("/bucket/folder1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/file1"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file0"));
    mFileSystem.createFile(new AlluxioURI("/bucket/folder0/file1"));
    // empty parameters
    ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
    List<URIStatus> statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    ListBucketResult expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults());
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", NO_PARAMS, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(6, expected.getContents().size());
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals("file1", expected.getContents().get(1).getKey());
    assertEquals("folder0/", expected.getContents().get(2).getKey());
    assertEquals("folder0/file0", expected.getContents().get(3).getKey());
    assertEquals("folder0/file1", expected.getContents().get(4).getKey());
    assertEquals("folder1/", expected.getContents().get(5).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with delimiter="/"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"));
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setDelimiter(AlluxioURI.SEPARATOR));
    Map<String, String> parameters = new HashMap<>();
    parameters.put("delimiter", AlluxioURI.SEPARATOR);
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(2, expected.getContents().size());
    assertEquals("file0", expected.getContents().get(0).getKey());
    assertEquals("file1", expected.getContents().get(1).getKey());
    assertEquals(2, expected.getCommonPrefixes().size());
    assertEquals("folder0/", expected.getCommonPrefixes().get(0).getPrefix());
    assertEquals("folder1/", expected.getCommonPrefixes().get(1).getPrefix());
    // parameters with prefix="folder0"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setPrefix("folder0"));
    parameters.clear();
    parameters.put("prefix", "folder0");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(3, expected.getContents().size());
    assertEquals("folder0/", expected.getContents().get(0).getKey());
    assertEquals("folder0/file0", expected.getContents().get(1).getKey());
    assertEquals("folder0/file1", expected.getContents().get(2).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
    // parameters with list-type=2 start-after="folder0/file0"
    statuses = mFileSystem.listStatus(new AlluxioURI("/bucket"), options);
    expected = new ListBucketResult("bucket", statuses, ListBucketOptions.defaults().setListType(2).setStartAfter("file0"));
    parameters.clear();
    parameters.put("list-type", "2");
    parameters.put("start-after", "file0");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/bucket", parameters, HttpMethod.GET, expected, TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE)).run();
    assertEquals(5, expected.getContents().size());
    assertEquals("file1", expected.getContents().get(0).getKey());
    assertEquals("folder0/", expected.getContents().get(1).getKey());
    assertEquals("folder0/file0", expected.getContents().get(2).getKey());
    assertEquals("folder0/file1", expected.getContents().get(3).getKey());
    assertEquals("folder1/", expected.getContents().get(4).getKey());
    assertEquals(0, expected.getCommonPrefixes().size());
}