Examples with ListObjectsRequest com.amazonaws.s3.model.ListObjectsRequest used on opensource projects

Example 96 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project gretl by sogis.

the class S3UploadStepTest method uploadDirectory_Ok.

@Test
@Category(S3Test.class)
public void uploadDirectory_Ok() throws Exception {
    File sourceObject = new File("src/test/resources/data/s3upload/");
    String s3EndPoint = "https://s3.eu-central-1.amazonaws.com";
    String s3Region = "eu-central-1";
    String acl = "public-read";
    Map<String, String> metaData = new HashMap<String, String>();
    metaData.put("lastModified", "2020-08-28");
    AwsCredentialsProvider creds = StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKey, s3SecretKey));
    Region region = Region.of(s3Region);
    S3Client s3client = S3Client.builder().credentialsProvider(creds).region(region).endpointOverride(URI.create(s3EndPoint)).build();
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.txt").build());
    // Upload files from a directory.
    S3UploadStep s3UploadStep = new S3UploadStep();
    s3UploadStep.execute(s3AccessKey, s3SecretKey, sourceObject, s3BucketName, s3EndPoint, s3Region, acl, null, metaData);
    // Check result.
    ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(s3BucketName).build();
    ListObjectsResponse res = s3client.listObjects(listObjects);
    List<S3Object> objects = res.contents();
    List<String> keyList = new ArrayList<String>();
    for (ListIterator<S3Object> iterVals = objects.listIterator(); iterVals.hasNext(); ) {
        S3Object myValue = iterVals.next();
        keyList.add(myValue.key());
    }
    assertTrue(keyList.contains("foo.txt"));
    assertTrue(keyList.contains("bar.txt"));
    // Remove uploaded files from bucket.
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.txt").build());
}

Example 97 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project gretl by sogis.

the class S3Bucket2BucketStep method execute.

public void execute(String accessKey, String secretKey, String sourceBucket, String targetBucket, String s3EndPoint, String s3Region, String acl, Map<String, String> metaData) throws FileNotFoundException, UnsupportedEncodingException {
    log.lifecycle(String.format("Start S3Bucket2Bucket(Name: %s SourceBucket: %s TargetBucket: %s S3EndPoint: %s S3Region: %s ACL: %s MetaData: %s)", taskName, sourceBucket, targetBucket, s3EndPoint, s3Region, acl, metaData));
    AwsCredentialsProvider creds = StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey));
    Region region = Region.of(s3Region);
    S3Client s3client = S3Client.builder().credentialsProvider(creds).region(region).endpointOverride(URI.create(s3EndPoint)).build();
    int copiedFiles = 0;
    ObjectCannedACL aclObj = ObjectCannedACL.fromValue(acl);
    ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(sourceBucket).build();
    ListObjectsResponse res = s3client.listObjects(listObjects);
    List<S3Object> objects = res.contents();
    List<String> keyList = new ArrayList<String>();
    for (ListIterator<S3Object> iterVals = objects.listIterator(); iterVals.hasNext(); ) {
        S3Object myObject = iterVals.next();
        String encodedUrl = encodedUrl = URLEncoder.encode(sourceBucket + "/" + myObject.key(), StandardCharsets.UTF_8.toString());
        CopyObjectRequest copyReq = CopyObjectRequest.builder().copySource(encodedUrl).destinationBucket(targetBucket).destinationKey(myObject.key()).acl(aclObj).build();
        CopyObjectResponse copyRes = s3client.copyObject(copyReq);
        copiedFiles++;
    }
    log.lifecycle(taskName + ": " + copiedFiles + " Files have been copied: " + sourceBucket + " -> " + targetBucket);
}

Example 98 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project gretl by sogis.

the class S3UploadTest method uploadFileTree_Ok.

@Test
@Category(S3Test.class)
public void uploadFileTree_Ok() throws Exception {
    // Check result.
    AwsCredentialsProvider creds = StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKey, s3SecretKey));
    Region region = Region.of("eu-central-1");
    S3Client s3client = S3Client.builder().credentialsProvider(creds).region(region).endpointOverride(URI.create("https://s3.eu-central-1.amazonaws.com")).build();
    // Remove uploaded files from bucket.
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.csv").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.csv").build());
    // Upload all files from a directory.
    GradleVariable[] gvs = { GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey), GradleVariable.newGradleProperty("s3SecretKey", s3SecretKey), GradleVariable.newGradleProperty("s3BucketName", s3BucketName) };
    IntegrationTestUtil.runJob("src/integrationTest/jobs/S3UploadFileTree", gvs);
    ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(s3BucketName).build();
    ListObjectsResponse res = s3client.listObjects(listObjects);
    List<S3Object> objects = res.contents();
    List<String> keyList = new ArrayList<String>();
    for (ListIterator<S3Object> iterVals = objects.listIterator(); iterVals.hasNext(); ) {
        S3Object myValue = iterVals.next();
        keyList.add(myValue.key());
    }
    assertTrue(keyList.contains("foo.csv"));
    assertTrue(keyList.contains("bar.csv"));
    // TODO 3 wegen download.txt
    assertTrue(keyList.size() == 3);
    // Remove uploaded files from bucket.
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.csv").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.csv").build());
}

Example 99 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project gretl by sogis.

the class S3UploadTest method uploadDirectory_Ok.

@Test
@Category(S3Test.class)
public void uploadDirectory_Ok() throws Exception {
    AwsCredentialsProvider creds = StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKey, s3SecretKey));
    Region region = Region.of("eu-central-1");
    S3Client s3client = S3Client.builder().credentialsProvider(creds).region(region).endpointOverride(URI.create("https://s3.eu-central-1.amazonaws.com")).build();
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.txt").build());
    // Upload all files from a directory.
    GradleVariable[] gvs = { GradleVariable.newGradleProperty("s3AccessKey", s3AccessKey), GradleVariable.newGradleProperty("s3SecretKey", s3SecretKey), GradleVariable.newGradleProperty("s3BucketName", s3BucketName) };
    IntegrationTestUtil.runJob("src/integrationTest/jobs/S3UploadDirectory", gvs);
    // Check result.
    ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(s3BucketName).build();
    ListObjectsResponse res = s3client.listObjects(listObjects);
    List<S3Object> objects = res.contents();
    List<String> keyList = new ArrayList<String>();
    for (ListIterator<S3Object> iterVals = objects.listIterator(); iterVals.hasNext(); ) {
        S3Object myValue = iterVals.next();
        keyList.add(myValue.key());
    }
    assertTrue(keyList.contains("foo.txt"));
    assertTrue(keyList.contains("bar.txt"));
    // Remove uploaded files from bucket.
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("foo.txt").build());
    s3client.deleteObject(DeleteObjectRequest.builder().bucket(s3BucketName).key("bar.txt").build());
}

Example 100 with ListObjectsRequest

use of com.amazonaws.s3.model.ListObjectsRequest in project aws-sdk-android by aws-amplify.

the class S3IntegrationTest method testListObjects.

/**
 * Tests that we can list the objects in our test bucket.
 */
@Test
public void testListObjects() {
    List<S3ObjectSummary> objects = s3.listObjects(EXPECTED_BUCKET_NAME).getObjectSummaries();
    assertTrue(objectListContainsKey(objects, expectedKey));
    objects = s3.listObjects(EXPECTED_BUCKET_NAME, "non-existant-object-key-prefix-").getObjectSummaries();
    assertNotNull(objects);
    objects = s3.listObjects(new ListObjectsRequest(EXPECTED_BUCKET_NAME, null, null, null, 0)).getObjectSummaries();
    assertTrue(objects.size() == 0);
}