Examples with ListObjectsRequest com.amazonaws.s3.model.ListObjectsRequest used on opensource projects

Example 11 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project presto by prestodb.

the class TestPrestoS3FileSystem method testListPrefixModes.

@Test
public void testListPrefixModes() throws Exception {
    S3ObjectSummary rootObject = new S3ObjectSummary();
    rootObject.setStorageClass(StorageClass.Standard.toString());
    rootObject.setKey("standard-object-at-root.txt");
    rootObject.setLastModified(new Date());
    S3ObjectSummary childObject = new S3ObjectSummary();
    childObject.setStorageClass(StorageClass.Standard.toString());
    childObject.setKey("prefix/child-object.txt");
    childObject.setLastModified(new Date());
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3() {

            @Override
            public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) {
                ObjectListing listing = new ObjectListing();
                // Shallow listing
                if ("/".equals(listObjectsRequest.getDelimiter())) {
                    listing.getCommonPrefixes().add("prefix");
                    listing.getObjectSummaries().add(rootObject);
                    return listing;
                }
                // Recursive listing of object keys only
                listing.getObjectSummaries().addAll(Arrays.asList(childObject, rootObject));
                return listing;
            }
        };
        Path rootPath = new Path("s3n://test-bucket/");
        fs.initialize(rootPath.toUri(), new Configuration());
        fs.setS3Client(s3);
        List<LocatedFileStatus> shallowAll = remoteIteratorToList(fs.listLocatedStatus(rootPath));
        assertEquals(shallowAll.size(), 2);
        assertTrue(shallowAll.get(0).isDirectory());
        assertFalse(shallowAll.get(1).isDirectory());
        assertEquals(shallowAll.get(0).getPath(), new Path(rootPath, "prefix"));
        assertEquals(shallowAll.get(1).getPath(), new Path(rootPath, rootObject.getKey()));
        List<LocatedFileStatus> shallowFiles = remoteIteratorToList(fs.listFiles(rootPath, false));
        assertEquals(shallowFiles.size(), 1);
        assertFalse(shallowFiles.get(0).isDirectory());
        assertEquals(shallowFiles.get(0).getPath(), new Path(rootPath, rootObject.getKey()));
        List<LocatedFileStatus> recursiveFiles = remoteIteratorToList(fs.listFiles(rootPath, true));
        assertEquals(recursiveFiles.size(), 2);
        assertFalse(recursiveFiles.get(0).isDirectory());
        assertFalse(recursiveFiles.get(1).isDirectory());
        assertEquals(recursiveFiles.get(0).getPath(), new Path(rootPath, childObject.getKey()));
        assertEquals(recursiveFiles.get(1).getPath(), new Path(rootPath, rootObject.getKey()));
    }
}

Example 12 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project presto by prestodb.

the class PrestoS3FileSystem method listPrefix.

private Iterator<LocatedFileStatus> listPrefix(Path path, OptionalInt initialMaxKeys, ListingMode mode) {
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(getBucketName(uri)).withPrefix(key).withDelimiter(mode == ListingMode.RECURSIVE_FILES_ONLY ? null : PATH_SEPARATOR).withMaxKeys(initialMaxKeys.isPresent() ? initialMaxKeys.getAsInt() : null);
    STATS.newListObjectsCall();
    Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) {

        @Override
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            // Clear any max keys set for the initial request before submitting subsequent requests. Values < 0
            // are not sent in the request and the default limit is used
            previous.setMaxKeys(-1);
            return s3.listNextBatchOfObjects(previous);
        }
    };
    Iterator<LocatedFileStatus> result = Iterators.concat(Iterators.transform(listings, this::statusFromListing));
    if (mode.isFilesOnly()) {
        // Even recursive listing can still contain empty "directory" objects, must filter them out
        result = Iterators.filter(result, LocatedFileStatus::isFile);
    }
    return result;
}

Example 13 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project cloudstack by apache.

the class S3Utils method listDirectory.

public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
    LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));
    List<S3ObjectSummary> objects = new ArrayList<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.withBucketName(bucketName);
    listObjectsRequest.withPrefix(directory);
    ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
    if (ol.isTruncated()) {
        do {
            objects.addAll(ol.getObjectSummaries());
            listObjectsRequest.setMarker(ol.getNextMarker());
            ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
        } while (ol.isTruncated());
    } else {
        objects.addAll(ol.getObjectSummaries());
    }
    if (objects.isEmpty()) {
        return emptyList();
    }
    return unmodifiableList(objects);
}

Example 14 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project zeppelin by apache.

the class S3NotebookRepo method move.

@Override
public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) throws IOException {
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(rootFolder + folderPath + "/");
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().endsWith(".zpln")) {
                    String noteId = getNoteId(objectSummary.getKey());
                    String notePath = getNotePath(rootFolder, objectSummary.getKey());
                    String newNotePath = newFolderPath + notePath.substring(folderPath.length());
                    move(noteId, notePath, newNotePath, subject);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Fail to move folder: " + folderPath + " to " + newFolderPath + " in S3", ace);
    }
}

Example 15 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project zeppelin by apache.

the class S3NotebookRepo method list.

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
    Map<String, NoteInfo> notesInfo = new HashMap<>();
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().endsWith(".zpln")) {
                    try {
                        NoteInfo info = getNoteInfo(objectSummary.getKey());
                        notesInfo.put(info.getId(), info);
                    } catch (IOException e) {
                        LOGGER.warn(e.getMessage());
                    }
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Fail to list objects in S3", ace);
    }
    return notesInfo;
}