Examples with ListObjectsRequest com.amazonaws.services.s3.model.ListObjectsRequest used on opensource projects

Example 16 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project camel by apache.

the class S3Endpoint method doStart.

@Override
public void doStart() throws Exception {
    super.doStart();
    s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : createS3Client();
    if (ObjectHelper.isNotEmpty(configuration.getAmazonS3Endpoint())) {
        s3Client.setEndpoint(configuration.getAmazonS3Endpoint());
    }
    String fileName = getConfiguration().getFileName();
    if (fileName != null) {
        LOG.trace("File name [{}] requested, so skipping bucket check...", fileName);
        return;
    }
    String bucketName = getConfiguration().getBucketName();
    LOG.trace("Querying whether bucket [{}] already exists...", bucketName);
    String prefix = getConfiguration().getPrefix();
    try {
        s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0));
        LOG.trace("Bucket [{}] already exists", bucketName);
        return;
    } catch (AmazonServiceException ase) {
        /* 404 means the bucket doesn't exist */
        if (ase.getStatusCode() != 404) {
            throw ase;
        }
    }
    LOG.trace("Bucket [{}] doesn't exist yet", bucketName);
    // creates the new bucket because it doesn't exist yet
    CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName());
    if (getConfiguration().getRegion() != null) {
        createBucketRequest.setRegion(getConfiguration().getRegion());
    }
    LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest);
    s3Client.createBucket(createBucketRequest);
    LOG.trace("Bucket created");
    if (configuration.getPolicy() != null) {
        LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy());
        s3Client.setBucketPolicy(bucketName, configuration.getPolicy());
        LOG.trace("Bucket policy updated");
    }
}

Example 17 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project camel by apache.

the class AmazonS3ClientMock method listObjects.

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex;
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }
    return objectListing;
}

Example 18 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project hadoop by apache.

the class S3AFileSystem method innerDelete.

/**
   * Delete an object. See {@link #delete(Path, boolean)}.
   *
   * @param status fileStatus object
   * @param recursive if path is a directory and set to
   * true, the directory is deleted else throws an exception. In
   * case of a file the recursive can be set to either true or false.
   * @return  true if delete is successful else false.
   * @throws IOException due to inability to delete a directory or file.
   * @throws AmazonClientException on failures inside the AWS SDK
   */
private boolean innerDelete(S3AFileStatus status, boolean recursive) throws IOException, AmazonClientException {
    Path f = status.getPath();
    LOG.debug("Delete path {} - recursive {}", f, recursive);
    String key = pathToKey(f);
    if (status.isDirectory()) {
        LOG.debug("delete: Path is a directory: {}", f);
        if (!key.endsWith("/")) {
            key = key + "/";
        }
        if (key.equals("/")) {
            return rejectRootDirectoryDelete(status, recursive);
        }
        if (!recursive && !status.isEmptyDirectory()) {
            throw new PathIsNotEmptyDirectoryException(f.toString());
        }
        if (status.isEmptyDirectory()) {
            LOG.debug("Deleting fake empty directory {}", key);
            deleteObject(key);
            instrumentation.directoryDeleted();
        } else {
            LOG.debug("Getting objects for directory prefix {} to delete", key);
            ListObjectsRequest request = createListObjectsRequest(key, null);
            ObjectListing objects = listObjects(request);
            List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(objects.getObjectSummaries().size());
            while (true) {
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                    LOG.debug("Got object to delete {}", summary.getKey());
                    if (keys.size() == MAX_ENTRIES_TO_DELETE) {
                        removeKeys(keys, true, false);
                    }
                }
                if (objects.isTruncated()) {
                    objects = continueListObjects(objects);
                } else {
                    if (!keys.isEmpty()) {
                        removeKeys(keys, false, false);
                    }
                    break;
                }
            }
        }
    } else {
        LOG.debug("delete: Path is a file");
        instrumentation.fileDeleted(1);
        deleteObject(key);
    }
    Path parent = f.getParent();
    if (parent != null) {
        createFakeDirectoryIfNecessary(parent);
    }
    return true;
}

Example 19 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project hadoop by apache.

the class S3AFileSystem method getFileStatus.

/**
   * Return a file status object that represents the path.
   * @param f The path we want information from
   * @return a FileStatus object
   * @throws java.io.FileNotFoundException when the path does not exist;
   * @throws IOException on other problems.
   */
public S3AFileStatus getFileStatus(final Path f) throws IOException {
    incrementStatistic(INVOCATION_GET_FILE_STATUS);
    final Path path = qualify(f);
    String key = pathToKey(path);
    LOG.debug("Getting path status for {}  ({})", path, key);
    if (!key.isEmpty()) {
        try {
            ObjectMetadata meta = getObjectMetadata(key);
            if (objectRepresentsDirectory(key, meta.getContentLength())) {
                LOG.debug("Found exact file: fake directory");
                return new S3AFileStatus(true, path, username);
            } else {
                LOG.debug("Found exact file: normal file");
                return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
            }
        } catch (AmazonServiceException e) {
            if (e.getStatusCode() != 404) {
                throw translateException("getFileStatus", path, e);
            }
        } catch (AmazonClientException e) {
            throw translateException("getFileStatus", path, e);
        }
        // Necessary?
        if (!key.endsWith("/")) {
            String newKey = key + "/";
            try {
                ObjectMetadata meta = getObjectMetadata(newKey);
                if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
                    LOG.debug("Found file (with /): fake directory");
                    return new S3AFileStatus(true, path, username);
                } else {
                    LOG.warn("Found file (with /): real file? should not happen: {}", key);
                    return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
                }
            } catch (AmazonServiceException e) {
                if (e.getStatusCode() != 404) {
                    throw translateException("getFileStatus", newKey, e);
                }
            } catch (AmazonClientException e) {
                throw translateException("getFileStatus", newKey, e);
            }
        }
    }
    try {
        key = maybeAddTrailingSlash(key);
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(1);
        ObjectListing objects = listObjects(request);
        if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Found path as directory (with /): {}/{}", objects.getCommonPrefixes().size(), objects.getObjectSummaries().size());
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    LOG.debug("Summary: {} {}", summary.getKey(), summary.getSize());
                }
                for (String prefix : objects.getCommonPrefixes()) {
                    LOG.debug("Prefix: {}", prefix);
                }
            }
            return new S3AFileStatus(false, path, username);
        } else if (key.isEmpty()) {
            LOG.debug("Found root directory");
            return new S3AFileStatus(true, path, username);
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404) {
            throw translateException("getFileStatus", key, e);
        }
    } catch (AmazonClientException e) {
        throw translateException("getFileStatus", key, e);
    }
    LOG.debug("Not Found: {}", path);
    throw new FileNotFoundException("No such file or directory: " + path);
}

Example 20 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project ice by Netflix.

the class AwsUtils method listAllObjects.

/**
     * List all object summary with given prefix in the s3 bucket.
     * @param bucket
     * @param prefix
     * @return
     */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;
    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig);
        }
        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());
        } while (page.isTruncated());
        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}