Examples with ListObjectsRequest software.amazon.awssdk.services.s3.model.ListObjectsRequest used on opensource projects

Example 31 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project incubator-gobblin by apache.

the class AWSSdkClient method listS3Bucket.

/**
 * Get list of S3 objects within a S3 bucket qualified by prefix path
 *
 * @param bucketName S3 bucket name
 * @param prefix S3 prefix to object
 * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path
 */
public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) {
    final AmazonS3 amazonS3 = getS3Client();
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
    LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing);
    return objectListing.getObjectSummaries();
}

Example 32 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project zeppelin by apache.

the class S3NotebookRepo method remove.

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
    String key = user + "/" + "notebook" + "/" + noteId;
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
    try {
        ObjectListing objects = s3client.listObjects(listObjectsRequest);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                s3client.deleteObject(bucketName, objectSummary.getKey());
            }
            objects = s3client.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
}

Example 33 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project camel by apache.

the class S3Consumer method poll.

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }
        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }
        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}

Example 34 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project camel by apache.

the class S3Endpoint method doStart.

@Override
public void doStart() throws Exception {
    super.doStart();
    s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : createS3Client();
    if (ObjectHelper.isNotEmpty(configuration.getAmazonS3Endpoint())) {
        s3Client.setEndpoint(configuration.getAmazonS3Endpoint());
    }
    String fileName = getConfiguration().getFileName();
    if (fileName != null) {
        LOG.trace("File name [{}] requested, so skipping bucket check...", fileName);
        return;
    }
    String bucketName = getConfiguration().getBucketName();
    LOG.trace("Querying whether bucket [{}] already exists...", bucketName);
    String prefix = getConfiguration().getPrefix();
    try {
        s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0));
        LOG.trace("Bucket [{}] already exists", bucketName);
        return;
    } catch (AmazonServiceException ase) {
        /* 404 means the bucket doesn't exist */
        if (ase.getStatusCode() != 404) {
            throw ase;
        }
    }
    LOG.trace("Bucket [{}] doesn't exist yet", bucketName);
    // creates the new bucket because it doesn't exist yet
    CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName());
    if (getConfiguration().getRegion() != null) {
        createBucketRequest.setRegion(getConfiguration().getRegion());
    }
    LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest);
    s3Client.createBucket(createBucketRequest);
    LOG.trace("Bucket created");
    if (configuration.getPolicy() != null) {
        LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy());
        s3Client.setBucketPolicy(bucketName, configuration.getPolicy());
        LOG.trace("Bucket policy updated");
    }
}

Example 35 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project jackrabbit-oak by apache.

the class S3Backend method getAllMetadataRecords.

public List<DataRecord> getAllMetadataRecords(String prefix) {
    List<DataRecord> metadataList = new ArrayList<DataRecord>();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
        for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            metadataList.add(new S3DataRecord(s3service, bucket, stripMetaKeyPrefix(s3ObjSumm.getKey()), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true));
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return metadataList;
}