Examples with VersionListing com.amazonaws.services.s3.model.VersionListing used on opensource projects

Example 1 with VersionListing

use of com.amazonaws.services.s3.model.VersionListing in project nifi by apache.

the class TestListS3 method testListVersions.

@Test
public void testListVersions() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.USE_VERSIONS, "true");
    Date lastModified = new Date();
    VersionListing versionListing = new VersionListing();
    S3VersionSummary versionSummary1 = new S3VersionSummary();
    versionSummary1.setBucketName("test-bucket");
    versionSummary1.setKey("test-key");
    versionSummary1.setVersionId("1");
    versionSummary1.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary1);
    S3VersionSummary versionSummary2 = new S3VersionSummary();
    versionSummary2.setBucketName("test-bucket");
    versionSummary2.setKey("test-key");
    versionSummary2.setVersionId("2");
    versionSummary2.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary2);
    Mockito.when(mockS3Client.listVersions(Mockito.any(ListVersionsRequest.class))).thenReturn(versionListing);
    runner.run();
    ArgumentCaptor<ListVersionsRequest> captureRequest = ArgumentCaptor.forClass(ListVersionsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listVersions(captureRequest.capture());
    ListVersionsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listObjects(Mockito.any(ListObjectsRequest.class));
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "test-key");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    ff0.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff0.assertAttributeEquals("s3.version", "1");
    MockFlowFile ff1 = flowFiles.get(1);
    ff1.assertAttributeEquals("filename", "test-key");
    ff1.assertAttributeEquals("s3.bucket", "test-bucket");
    ff1.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff1.assertAttributeEquals("s3.version", "2");
}

Example 2 with VersionListing

use of com.amazonaws.services.s3.model.VersionListing in project herd by FINRAOS.

the class S3DaoTest method testDeleteDirectoryAssertHandleAmazonClientException.

@Test
public void testDeleteDirectoryAssertHandleAmazonClientException() {
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    try {
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName("s3BucketName");
        s3FileTransferRequestParamsDto.setS3KeyPrefix("s3KeyPrefix");
        VersionListing versionListing = new VersionListing();
        versionListing.getVersionSummaries().add(new S3VersionSummary());
        when(mockS3Operations.listVersions(any(), any())).thenReturn(versionListing);
        when(mockS3Operations.deleteObjects(any(), any())).thenThrow(new AmazonClientException("message"));
        try {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
            fail();
        } catch (Exception e) {
            assertEquals(IllegalStateException.class, e.getClass());
            assertEquals("Failed to delete keys/key versions with prefix \"s3KeyPrefix\" from bucket \"s3BucketName\". Reason: message", e.getMessage());
        }
    } finally {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

Example 3 with VersionListing

use of com.amazonaws.services.s3.model.VersionListing in project opencast by opencast.

the class AwsS3DistributionServiceImpl method restoreElement.

protected MediaPackageElement restoreElement(String channelId, MediaPackage mediaPackage, String elementId, String fileName) throws DistributionException {
    String objectName = null;
    if (StringUtils.isNotBlank(fileName)) {
        objectName = buildObjectName(channelId, mediaPackage.getIdentifier().toString(), elementId, fileName);
    } else {
        objectName = buildObjectName(channelId, mediaPackage.getIdentifier().toString(), mediaPackage.getElementById(elementId));
    }
    // Get the latest version of the file
    // Note that this should be the delete marker for the file. We'll check, but if there is more than one delete marker
    // we'll have probs
    ListVersionsRequest lv = new ListVersionsRequest().withBucketName(bucketName).withPrefix(objectName).withMaxResults(1);
    VersionListing listing = s3.listVersions(lv);
    if (listing.getVersionSummaries().size() < 1) {
        throw new DistributionException("Object not found: " + objectName);
    }
    String versionId = listing.getVersionSummaries().get(0).getVersionId();
    // Verify that this is in fact a delete marker
    GetObjectMetadataRequest metadata = new GetObjectMetadataRequest(bucketName, objectName, versionId);
    // Ok, so there's no way of asking AWS directly if the object is deleted in this version of the SDK
    // So instead, we ask for its metadata
    // If it's deleted, then there *isn't* any metadata and we get a 404, which throws the exception
    // This, imo, is an incredibly boneheaded omission from the AWS SDK, and implies we should look for something which
    // sucks less
    // FIXME: This section should be refactored with a simple s3.doesObjectExist(bucketName, objectName) once we update
    // the AWS SDK
    boolean isDeleted = false;
    try {
        s3.getObjectMetadata(metadata);
    } catch (AmazonServiceException e) {
        // Note: This exception is actually a 405, not a 404.
        // This is expected, but very confusing if you're thinking it should be a 'file not found', rather than a 'method
        // not allowed on stuff that's deleted'
        // It's unclear what the expected behaviour is for things which have never existed...
        isDeleted = true;
    }
    if (isDeleted) {
        // Delete the delete marker
        DeleteVersionRequest delete = new DeleteVersionRequest(bucketName, objectName, versionId);
        s3.deleteVersion(delete);
    }
    return mediaPackage.getElementById(elementId);
}

Example 4 with VersionListing

use of com.amazonaws.services.s3.model.VersionListing in project aws-doc-sdk-examples by awsdocs.

the class DeleteBucket method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n" + "Ex: DeleteBucket <bucketname>\n";
    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }
            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext(); ) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }
            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }
        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

Example 5 with VersionListing

use of com.amazonaws.services.s3.model.VersionListing in project aws-doc-sdk-examples by awsdocs.

the class DeleteBucket2 method main.

public static void main(String[] args) {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Delete all objects from the bucket. This is sufficient
        // for unversioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts
        // delete markers for all objects, but doesn't delete the object versions.
        // To delete objects from versioned buckets, delete all of the object versions before deleting
        // the bucket (see below for an example).
        ObjectListing objectListing = s3Client.listObjects(bucketName);
        while (true) {
            Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
            while (objIter.hasNext()) {
                s3Client.deleteObject(bucketName, objIter.next().getKey());
            }
            // and delete them.
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        // Delete all object versions (required for versioned buckets).
        VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
        while (true) {
            Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
            while (versionIter.hasNext()) {
                S3VersionSummary vs = versionIter.next();
                s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
            }
            if (versionList.isTruncated()) {
                versionList = s3Client.listNextBatchOfVersions(versionList);
            } else {
                break;
            }
        }
        // After all objects and object versions are deleted, delete the bucket.
        s3Client.deleteBucket(bucketName);
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client couldn't
        // parse the response from Amazon S3.
        e.printStackTrace();
    }
}