Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 36 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project molgenis by molgenis.

the class AmazonBucketClientImplTest method downloadFileExpression.

@Test
public void downloadFileExpression() throws IOException {
    ObjectListing objectListing = mock(ObjectListing.class);
    S3ObjectSummary s3ObjectSummary1 = mock(S3ObjectSummary.class);
    S3ObjectSummary s3ObjectSummary2 = mock(S3ObjectSummary.class);
    S3ObjectSummary s3ObjectSummary3 = mock(S3ObjectSummary.class);
    S3ObjectSummary s3ObjectSummary4 = mock(S3ObjectSummary.class);
    when(client.getObject(any())).thenReturn(s3Object);
    when(s3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(new FileInputStream(ResourceUtils.getFile(getClass(), "/test_data_only.xlsx")), httpRequestBase));
    when(client.listObjects("bucket")).thenReturn(objectListing);
    Calendar c1 = Calendar.getInstance();
    c1.set(2017, 2, 7);
    Calendar c2 = Calendar.getInstance();
    c2.set(2016, 2, 7);
    Calendar c3 = Calendar.getInstance();
    c3.set(2015, 2, 7);
    Calendar c4 = Calendar.getInstance();
    c4.set(2017, 3, 7);
    when(s3ObjectSummary1.getKey()).thenReturn("kie");
    when(s3ObjectSummary1.getLastModified()).thenReturn(c1.getTime());
    when(s3ObjectSummary2.getKey()).thenReturn("keye");
    when(s3ObjectSummary2.getLastModified()).thenReturn(c2.getTime());
    when(s3ObjectSummary3.getKey()).thenReturn("keyw");
    when(s3ObjectSummary3.getLastModified()).thenReturn(c3.getTime());
    when(s3ObjectSummary4.getKey()).thenReturn("keyq");
    when(s3ObjectSummary4.getLastModified()).thenReturn(c4.getTime());
    when(objectListing.getObjectSummaries()).thenReturn(Arrays.asList(s3ObjectSummary1, s3ObjectSummary2, s3ObjectSummary3, s3ObjectSummary4));
    amazonBucketClient.downloadFile(client, fileStore, "ID", "bucket", "key(.*)", null, true, null);
    verify(fileStore).store(any(), eq("bucket_ID" + File.separatorChar + "keyq.xlsx"));
}

Example 37 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project qpp-conversion-tool by CMSgov.

the class AwsTestHelper method cleanS3.

/**
 * Cleans up the test s3 bucket by batch for performance purposes
 */
public static void cleanS3() {
    ObjectListing objectListing = S3_CLIENT.listObjects(TEST_S3_BUCKET_NAME);
    boolean firstTimeThrough = true;
    do {
        if (!firstTimeThrough) {
            objectListing = S3_CLIENT.listNextBatchOfObjects(objectListing);
        }
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            S3_CLIENT.deleteObject(TEST_S3_BUCKET_NAME, objectSummary.getKey());
        }
        firstTimeThrough = false;
    } while (objectListing.isTruncated());
}

Example 38 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project presto by prestodb.

the class MockAmazonS3 method listObjects.

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) {
    ObjectListing listing = new ObjectListing();
    S3ObjectSummary standard = new S3ObjectSummary();
    standard.setStorageClass(StorageClass.Standard.toString());
    standard.setKey(STANDARD_OBJECT_KEY);
    standard.setLastModified(new Date());
    listing.getObjectSummaries().add(standard);
    if (hasHadoopFolderMarkerObjects) {
        S3ObjectSummary hadoopFolderMarker = new S3ObjectSummary();
        hadoopFolderMarker.setStorageClass(StorageClass.Standard.toString());
        hadoopFolderMarker.setKey("test/test_$folder$");
        hadoopFolderMarker.setLastModified(new Date());
        listing.getObjectSummaries().add(hadoopFolderMarker);
    }
    if (hasGlacierObjects) {
        S3ObjectSummary glacier = new S3ObjectSummary();
        glacier.setStorageClass(StorageClass.Glacier.toString());
        glacier.setKey(GLACIER_OBJECT_KEY);
        glacier.setLastModified(new Date());
        listing.getObjectSummaries().add(glacier);
    }
    return listing;
}

Example 39 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project presto by prestodb.

the class TestPrestoS3FileSystem method testListPrefixModes.

@Test
public void testListPrefixModes() throws Exception {
    S3ObjectSummary rootObject = new S3ObjectSummary();
    rootObject.setStorageClass(StorageClass.Standard.toString());
    rootObject.setKey("standard-object-at-root.txt");
    rootObject.setLastModified(new Date());
    S3ObjectSummary childObject = new S3ObjectSummary();
    childObject.setStorageClass(StorageClass.Standard.toString());
    childObject.setKey("prefix/child-object.txt");
    childObject.setLastModified(new Date());
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3() {

            @Override
            public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) {
                ObjectListing listing = new ObjectListing();
                // Shallow listing
                if ("/".equals(listObjectsRequest.getDelimiter())) {
                    listing.getCommonPrefixes().add("prefix");
                    listing.getObjectSummaries().add(rootObject);
                    return listing;
                }
                // Recursive listing of object keys only
                listing.getObjectSummaries().addAll(Arrays.asList(childObject, rootObject));
                return listing;
            }
        };
        Path rootPath = new Path("s3n://test-bucket/");
        fs.initialize(rootPath.toUri(), new Configuration());
        fs.setS3Client(s3);
        List<LocatedFileStatus> shallowAll = remoteIteratorToList(fs.listLocatedStatus(rootPath));
        assertEquals(shallowAll.size(), 2);
        assertTrue(shallowAll.get(0).isDirectory());
        assertFalse(shallowAll.get(1).isDirectory());
        assertEquals(shallowAll.get(0).getPath(), new Path(rootPath, "prefix"));
        assertEquals(shallowAll.get(1).getPath(), new Path(rootPath, rootObject.getKey()));
        List<LocatedFileStatus> shallowFiles = remoteIteratorToList(fs.listFiles(rootPath, false));
        assertEquals(shallowFiles.size(), 1);
        assertFalse(shallowFiles.get(0).isDirectory());
        assertEquals(shallowFiles.get(0).getPath(), new Path(rootPath, rootObject.getKey()));
        List<LocatedFileStatus> recursiveFiles = remoteIteratorToList(fs.listFiles(rootPath, true));
        assertEquals(recursiveFiles.size(), 2);
        assertFalse(recursiveFiles.get(0).isDirectory());
        assertFalse(recursiveFiles.get(1).isDirectory());
        assertEquals(recursiveFiles.get(0).getPath(), new Path(rootPath, childObject.getKey()));
        assertEquals(recursiveFiles.get(1).getPath(), new Path(rootPath, rootObject.getKey()));
    }
}

Example 40 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project presto by prestodb.

the class PrestoS3FileSystem method listPrefix.

private Iterator<LocatedFileStatus> listPrefix(Path path, OptionalInt initialMaxKeys, ListingMode mode) {
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(getBucketName(uri)).withPrefix(key).withDelimiter(mode == ListingMode.RECURSIVE_FILES_ONLY ? null : PATH_SEPARATOR).withMaxKeys(initialMaxKeys.isPresent() ? initialMaxKeys.getAsInt() : null);
    STATS.newListObjectsCall();
    Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) {

        @Override
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            // Clear any max keys set for the initial request before submitting subsequent requests. Values < 0
            // are not sent in the request and the default limit is used
            previous.setMaxKeys(-1);
            return s3.listNextBatchOfObjects(previous);
        }
    };
    Iterator<LocatedFileStatus> result = Iterators.concat(Iterators.transform(listings, this::statusFromListing));
    if (mode.isFilesOnly()) {
        // Even recursive listing can still contain empty "directory" objects, must filter them out
        result = Iterators.filter(result, LocatedFileStatus::isFile);
    }
    return result;
}