Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 26 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project incubator-gobblin by apache.

the class AWSSdkClient method listS3Bucket.

/**
 * Get list of S3 objects within a S3 bucket qualified by prefix path
 *
 * @param bucketName S3 bucket name
 * @param prefix S3 prefix to object
 * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path
 */
public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) {
    final AmazonS3 amazonS3 = getS3Client();
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
    LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing);
    return objectListing.getObjectSummaries();
}

Example 27 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method renameKeys.

/**
 * This method rename object keys in S3 concurrently. The number of
 * concurrent threads is defined by 'maxConnections' property in
 * aws.properties. As S3 doesn't have "move" command, this method simulate
 * move as copy object object to new key and then delete older key.
 */
private void renameKeys() throws DataStoreException {
    long startTime = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    long count = 0;
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
        ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
        boolean taskAdded = false;
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
                taskAdded = true;
                count++;
                // delete the object if it follows old key name format
                if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
            }
            if (!prevObjectListing.isTruncated())
                break;
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        // This will make the executor accept no new threads
        // and finish all existing threads in the queue
        executor.shutdown();
        try {
            // Wait until all threads are finish
            while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
                LOG.info("Rename S3 keys tasks timedout. Waiting again");
            }
        } catch (InterruptedException ie) {
        }
        LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
        // Delete older keys.
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            int batchSize = 500, startIndex = 0, size = deleteList.size();
            int endIndex = batchSize < size ? batchSize : size;
            while (endIndex <= size) {
                delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1));
                if (endIndex == size) {
                    break;
                } else {
                    startIndex = endIndex;
                    endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
                }
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 28 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class Utils method deleteBucket.

/**
 * Delete S3 bucket. This method first deletes all objects from bucket and
 * then delete empty bucket.
 *
 * @param bucketName the bucket name.
 */
public static void deleteBucket(final String bucketName) throws IOException {
    Properties prop = readConfig(DEFAULT_CONFIG_FILE);
    AmazonS3 s3service = openService(prop);
    ObjectListing prevObjectListing = s3service.listObjects(bucketName);
    while (true) {
        for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            s3service.deleteObject(bucketName, s3ObjSumm.getKey());
        }
        if (!prevObjectListing.isTruncated()) {
            break;
        }
        prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
    }
    s3service.deleteBucket(bucketName);
}

Example 29 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project herd by FINRAOS.

the class S3DaoTest method testListDirectoryAssertIgnoreDirectories.

@Test
public void testListDirectoryAssertIgnoreDirectories() {
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    try {
        String s3BucketName = "s3BucketName";
        String s3KeyPrefix = "s3KeyPrefix";
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);
        boolean ignoreZeroByteDirectoryMarkers = true;
        when(mockS3Operations.listObjects(any(), any())).then(new Answer<ObjectListing>() {

            @Override
            public ObjectListing answer(InvocationOnMock invocation) throws Throwable {
                ListObjectsRequest listObjectsRequest = invocation.getArgument(0);
                assertEquals(s3BucketName, listObjectsRequest.getBucketName());
                assertEquals(s3KeyPrefix, listObjectsRequest.getPrefix());
                ObjectListing objectListing = new ObjectListing();
                {
                    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                    s3ObjectSummary.setBucketName(s3BucketName);
                    s3ObjectSummary.setKey("valid/object/key");
                    s3ObjectSummary.setSize(1024l);
                    objectListing.getObjectSummaries().add(s3ObjectSummary);
                }
                {
                    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                    s3ObjectSummary.setBucketName(s3BucketName);
                    s3ObjectSummary.setKey("empty/file");
                    s3ObjectSummary.setSize(0l);
                    objectListing.getObjectSummaries().add(s3ObjectSummary);
                }
                {
                    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                    s3ObjectSummary.setBucketName(s3BucketName);
                    s3ObjectSummary.setKey("directory/path/");
                    s3ObjectSummary.setSize(0l);
                    objectListing.getObjectSummaries().add(s3ObjectSummary);
                }
                // directory with a non-zero size is impossible, but we have a conditional branch to cover
                {
                    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                    s3ObjectSummary.setBucketName(s3BucketName);
                    s3ObjectSummary.setKey("another/directory/path/");
                    s3ObjectSummary.setSize(1024l);
                    objectListing.getObjectSummaries().add(s3ObjectSummary);
                }
                return objectListing;
            }
        });
        List<S3ObjectSummary> s3ObjectSummaries = s3Dao.listDirectory(s3FileTransferRequestParamsDto, ignoreZeroByteDirectoryMarkers);
        assertEquals(3, s3ObjectSummaries.size());
        assertEquals("valid/object/key", s3ObjectSummaries.get(0).getKey());
        assertEquals(1024l, s3ObjectSummaries.get(0).getSize());
        assertEquals("empty/file", s3ObjectSummaries.get(1).getKey());
        assertEquals(0l, s3ObjectSummaries.get(1).getSize());
        assertEquals("another/directory/path/", s3ObjectSummaries.get(2).getKey());
        assertEquals(1024l, s3ObjectSummaries.get(2).getSize());
    } finally {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

Example 30 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project tutorials by eugenp.

the class S3Application method main.

public static void main(String[] args) throws IOException {
    // set-up the client
    AmazonS3 s3client = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).withRegion(Regions.US_EAST_2).build();
    AWSS3Service awsService = new AWSS3Service(s3client);
    bucketName = "baeldung-bucket";
    // creating a bucket
    if (awsService.doesBucketExist(bucketName)) {
        System.out.println("Bucket name is not available." + " Try again with a different Bucket name.");
        return;
    }
    awsService.createBucket(bucketName);
    // list all the buckets
    for (Bucket s : awsService.listBuckets()) {
        System.out.println(s.getName());
    }
    // deleting bucket
    awsService.deleteBucket("baeldung-bucket-test2");
    // uploading object
    awsService.putObject(bucketName, "Document/hello.txt", new File("/Users/user/Document/hello.txt"));
    // listing objects
    ObjectListing objectListing = awsService.listObjects(bucketName);
    for (S3ObjectSummary os : objectListing.getObjectSummaries()) {
        System.out.println(os.getKey());
    }
    // downloading an object
    S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt");
    S3ObjectInputStream inputStream = s3object.getObjectContent();
    FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt"));
    // copying an object
    awsService.copyObject("baeldung-bucket", "picture/pic.png", "baeldung-bucket2", "Document/picture.png");
    // deleting an object
    awsService.deleteObject(bucketName, "Document/hello.txt");
    // deleting multiple objects
    String[] objkeyArr = { "Document/hello2.txt", "Document/picture.png" };
    DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket").withKeys(objkeyArr);
    awsService.deleteObjects(delObjReq);
}