Examples with ListObjectsV2Request com.amazonaws.services.s3.model.ListObjectsV2Request used on opensource projects

Example 21 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project flyway by flyway.

the class AwsS3Scanner method scanForResources.

/**
 * Scans S3 for the resources. In AWS SDK v2, only the region that the client is configured with can be used.
 * The format of the path is expected to be {@code s3:{bucketName}/{optional prefix}}.
 *
 * @param location The location in S3 to start searching. Subdirectories are also searched.
 * @return The resources that were found.
 */
@Override
public Collection<LoadableResource> scanForResources(final Location location) {
    String bucketName = getBucketName(location);
    String prefix = getPrefix(bucketName, location.getPath());
    S3Client s3Client = S3ClientFactory.getClient();
    try {
        ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix);
        ListObjectsV2Request request = builder.build();
        ListObjectsV2Response listObjectResult = s3Client.listObjectsV2(request);
        return getLoadableResources(bucketName, listObjectResult);
    } catch (SdkClientException e) {
        if (throwOnMissingLocations) {
            throw new FlywayException("Could not access s3 location:" + bucketName + prefix + " due to error: " + e.getMessage());
        }
        LOG.error("Skipping s3 location:" + bucketName + prefix + " due to error: " + e.getMessage());
        return Collections.emptyList();
    }
}

Example 22 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project engine by craftercms.

the class S3SiteListResolver method getSiteListFromBucketKeys.

protected Collection<String> getSiteListFromBucketKeys(String bucketName, String rootPrefix) {
    List<String> siteNames = new ArrayList<>();
    AmazonS3 client = clientBuilder.getClient();
    ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(rootPrefix).withDelimiter(DELIMITER);
    ListObjectsV2Result result = client.listObjectsV2(request);
    if (CollectionUtils.isNotEmpty(result.getCommonPrefixes())) {
        result.getCommonPrefixes().stream().map(prefix -> StringUtils.stripEnd(StringUtils.removeStart(prefix, rootPrefix), DELIMITER)).forEach(siteNames::add);
    }
    return siteNames;
}

Example 23 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project alluxio by Alluxio.

the class S3AUnderFileSystem method getObjectListingChunk.

// Get next chunk of listing result.
private ListObjectsV2Result getObjectListingChunk(ListObjectsV2Request request) throws IOException {
    ListObjectsV2Result result;
    try {
        // Query S3 for the next batch of objects.
        result = mClient.listObjectsV2(request);
        // Advance the request continuation token to the next set of objects.
        request.setContinuationToken(result.getNextContinuationToken());
    } catch (AmazonClientException e) {
        throw new IOException(e);
    }
    return result;
}

Example 24 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project alluxio by Alluxio.

the class S3AUnderFileSystem method getObjectListingChunk.

@Override
@Nullable
protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException {
    String delimiter = recursive ? "" : PATH_SEPARATOR;
    key = PathUtils.normalizePath(key, PATH_SEPARATOR);
    // In case key is root (empty string) do not normalize prefix.
    key = key.equals(PATH_SEPARATOR) ? "" : key;
    if (mUfsConf.isSet(PropertyKey.UNDERFS_S3_LIST_OBJECTS_V1) && mUfsConf.getBoolean(PropertyKey.UNDERFS_S3_LIST_OBJECTS_V1)) {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucketName).withPrefix(key).withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mUfsConf));
        ObjectListing result = getObjectListingChunkV1(request);
        if (result != null) {
            return new S3AObjectListingChunkV1(request, result);
        }
    } else {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(key).withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mUfsConf));
        ListObjectsV2Result result = getObjectListingChunk(request);
        if (result != null) {
            return new S3AObjectListingChunk(request, result);
        }
    }
    return null;
}

Example 25 with ListObjectsV2Request

use of com.amazonaws.services.s3.model.ListObjectsV2Request in project druid by druid-io.

the class S3DataSegmentMover method selfCheckingMove.

/**
 * Copies an object and after that checks that the object is present at the target location, via a separate API call.
 * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check
 * is added after it was observed that S3 may report a successful move, and the object is not found at the target
 * location.
 */
private void selfCheckingMove(String s3Bucket, String targetS3Bucket, String s3Path, String targetS3Path, String copyMsg) throws IOException, SegmentLoadingException {
    if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) {
        log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path);
        return;
    }
    final ServerSideEncryptingAmazonS3 s3Client = this.s3ClientSupplier.get();
    if (s3Client.doesObjectExist(s3Bucket, s3Path)) {
        final ListObjectsV2Result listResult = s3Client.listObjectsV2(new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Path).withMaxKeys(1));
        // keyCount is still zero.
        if (listResult.getObjectSummaries().size() == 0) {
            // should never happen
            throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path);
        }
        final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0);
        if (objectSummary.getStorageClass() != null && StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass())).equals(StorageClass.Glacier)) {
            throw new AmazonServiceException(StringUtils.format("Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path));
        } else {
            log.info("Moving file %s", copyMsg);
            final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket, targetS3Path);
            if (!config.getDisableAcl()) {
                copyRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket));
            }
            s3Client.copyObject(copyRequest);
            if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
                throw new IOE("After copy was reported as successful the file doesn't exist in the target location [%s]", copyMsg);
            }
            deleteWithRetriesSilent(s3Bucket, s3Path);
            log.debug("Finished moving file %s", copyMsg);
        }
    } else {
        // ensure object exists in target location
        if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
            log.info("Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", s3Bucket, s3Path, targetS3Bucket, targetS3Path);
        } else {
            throw new SegmentLoadingException("Unable to move file %s, not present in either source or target location", copyMsg);
        }
    }
}