Search in sources :

Example 86 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project exhibitor by soabase.

the class TestS3BackupProviderBase method testGetAvailableBackupKeys.

@Test
public void testGetAvailableBackupKeys() throws Exception {
    ObjectListing listing = new ObjectListing() {

        @Override
        public List<S3ObjectSummary> getObjectSummaries() {
            List<S3ObjectSummary> list = Lists.newArrayList();
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "one" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "two" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            summary = new S3ObjectSummary();
            summary.setKey("exhibitor-backup" + S3BackupProvider.SEPARATOR + "three" + S3BackupProvider.SEPARATOR + "1234");
            list.add(summary);
            return list;
        }
    };
    MockS3Client s3Client = new MockS3Client(null, listing);
    S3BackupProvider provider = new S3BackupProvider(new MockS3ClientFactory(s3Client), new PropertyBasedS3Credential(new Properties()), new PropertyBasedS3ClientConfig(new Properties()), null);
    List<BackupMetaData> backups = provider.getAvailableBackups(null, Maps.<String, String>newHashMap());
    List<String> backupNames = Lists.transform(backups, new Function<BackupMetaData, String>() {

        @Override
        public String apply(BackupMetaData metaData) {
            return metaData.getName();
        }
    });
    Assert.assertEquals(backupNames, Arrays.asList("one", "two", "three"));
}
Also used : BackupMetaData(com.netflix.exhibitor.core.backup.BackupMetaData) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) PropertyBasedS3ClientConfig(com.netflix.exhibitor.core.s3.PropertyBasedS3ClientConfig) Properties(java.util.Properties) PropertyBasedS3Credential(com.netflix.exhibitor.core.s3.PropertyBasedS3Credential) Test(org.testng.annotations.Test)

Example 87 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project exhibitor by soabase.

the class S3PseudoLock method getFileNames.

@Override
protected List<String> getFileNames(String lockPrefix) throws Exception {
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);
    return Lists.transform(objectListing.getObjectSummaries(), new Function<S3ObjectSummary, String>() {

        @Override
        public String apply(S3ObjectSummary summary) {
            return summary.getKey();
        }
    });
}
Also used : ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary)

Example 88 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project crate by crate.

the class S3FileReadingCollectorTest method createBatchIterator.

private BatchIterator<Row> createBatchIterator(Collection<String> fileUris, String compression, final S3ObjectInputStream s3InputStream, boolean collectSourceUriFailure) {
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(txnCtx, FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = new ArrayList<>(2);
    Reference raw = createReference(SourceLineExpression.COLUMN_NAME, DataTypes.STRING);
    inputs.add(ctx.add(raw));
    if (collectSourceUriFailure) {
        Reference sourceUriFailure = createReference(SourceUriFailureExpression.COLUMN_NAME, DataTypes.STRING);
        // noinspection unchecked
        sourceUriFailureInput = (Input<String>) ctx.add(sourceUriFailure);
        inputs.add(sourceUriFailureInput);
    }
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, Map.of(S3FileInputFactory.NAME, new FileInputFactory() {

        @Override
        public FileInput create(URI uri, Settings withClauseOptions) throws IOException {
            return new S3FileInput(new S3ClientHelper() {

                @Override
                protected AmazonS3 initClient(String accessKey, String secretKey, String endpoint, String protocol) throws IOException {
                    AmazonS3 client = mock(AmazonS3Client.class);
                    ObjectListing objectListing = mock(ObjectListing.class);
                    S3ObjectSummary summary = mock(S3ObjectSummary.class);
                    S3Object s3Object = mock(S3Object.class);
                    when(client.listObjects(anyString(), anyString())).thenReturn(objectListing);
                    when(objectListing.getObjectSummaries()).thenReturn(Collections.singletonList(summary));
                    when(summary.getKey()).thenReturn("foo");
                    when(client.getObject("fakebucket", "foo")).thenReturn(s3Object);
                    when(s3Object.getObjectContent()).thenReturn(s3InputStream);
                    when(client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
                    when(objectListing.isTruncated()).thenReturn(false);
                    return client;
                }
            }, uri, "https");
        }
    }), false, 1, 0, CopyFromParserProperties.DEFAULT, FileUriCollectPhase.InputFormat.JSON, Settings.EMPTY);
}
Also used : FileInputFactory(io.crate.execution.engine.collect.files.FileInputFactory) InputFactory(io.crate.expression.InputFactory) AmazonS3(com.amazonaws.services.s3.AmazonS3) LineCollectorExpression(io.crate.execution.engine.collect.files.LineCollectorExpression) TestingHelpers.createReference(io.crate.testing.TestingHelpers.createReference) Reference(io.crate.metadata.Reference) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) IOException(java.io.IOException) URI(java.net.URI) Input(io.crate.data.Input) FileInput(io.crate.execution.engine.collect.files.FileInput) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) FileLineReferenceResolver(io.crate.expression.reference.file.FileLineReferenceResolver) S3ClientHelper(io.crate.copy.s3.common.S3ClientHelper) FileInputFactory(io.crate.execution.engine.collect.files.FileInputFactory) S3Object(com.amazonaws.services.s3.model.S3Object) Settings(org.elasticsearch.common.settings.Settings)

Example 89 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project crate by crate.

the class S3FileInputTest method testListListUrlsWhenEmptyKeysIsListed.

@Test
public void testListListUrlsWhenEmptyKeysIsListed() throws Exception {
    S3ObjectSummary path = new S3ObjectSummary();
    path.setBucketName(BUCKET_NAME);
    path.setKey("prefix/");
    listObjectSummaries = objectSummaries();
    listObjectSummaries.add(path);
    when(objectListing.getObjectSummaries()).thenReturn(listObjectSummaries);
    List<URI> uris = s3FileInput.expandUri();
    assertThat(uris.size(), is(2));
    assertThat(uris.get(0).toString(), is("s3:///fakeBucket/prefix/test1.json.gz"));
    assertThat(uris.get(1).toString(), is("s3:///fakeBucket/prefix/test2.json.gz"));
}
Also used : S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) S3URI(io.crate.copy.s3.common.S3URI) URI(java.net.URI) Test(org.junit.Test)

Example 90 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project crate by crate.

the class S3BlobContainer method delete.

@Override
public void delete() throws IOException {
    try (AmazonS3Reference clientReference = blobStore.clientReference()) {
        ObjectListing prevListing = null;
        while (true) {
            ObjectListing list;
            if (prevListing != null) {
                final ObjectListing finalPrevListing = prevListing;
                list = clientReference.client().listNextBatchOfObjects(finalPrevListing);
            } else {
                final ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
                listObjectsRequest.setBucketName(blobStore.bucket());
                listObjectsRequest.setPrefix(keyPath);
                list = clientReference.client().listObjects(listObjectsRequest);
            }
            final List<String> blobsToDelete = list.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toList());
            if (list.isTruncated()) {
                doDeleteBlobs(blobsToDelete, false);
                prevListing = list;
            } else {
                final List<String> lastBlobsToDelete = new ArrayList<>(blobsToDelete);
                lastBlobsToDelete.add(keyPath);
                doDeleteBlobs(lastBlobsToDelete, false);
                break;
            }
        }
    } catch (final AmazonClientException e) {
        throw new IOException("Exception when deleting blob container [" + keyPath + "]", e);
    }
}
Also used : ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) AmazonClientException(com.amazonaws.AmazonClientException) ArrayList(java.util.ArrayList) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) IOException(java.io.IOException)

Aggregations

S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)196 ObjectListing (com.amazonaws.services.s3.model.ObjectListing)106 ArrayList (java.util.ArrayList)64 ListObjectsRequest (com.amazonaws.services.s3.model.ListObjectsRequest)61 Test (org.junit.Test)50 Date (java.util.Date)29 DeleteObjectsRequest (com.amazonaws.services.s3.model.DeleteObjectsRequest)27 ListObjectsV2Result (com.amazonaws.services.s3.model.ListObjectsV2Result)25 Test (org.testng.annotations.Test)25 AmazonS3 (com.amazonaws.services.s3.AmazonS3)23 S3Object (com.amazonaws.services.s3.model.S3Object)19 AmazonClientException (com.amazonaws.AmazonClientException)18 IOException (java.io.IOException)17 S3FileTransferRequestParamsDto (org.finra.herd.model.dto.S3FileTransferRequestParamsDto)16 AmazonServiceException (com.amazonaws.AmazonServiceException)14 ListObjectsV2Request (com.amazonaws.services.s3.model.ListObjectsV2Request)14 File (java.io.File)13 HashMap (java.util.HashMap)13 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)13 StorageFile (org.finra.herd.model.api.xml.StorageFile)13