Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 66 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project deeplearning4j by deeplearning4j.

the class S3Downloader method paginate.

/**
     * Paginates through a bucket's keys invoking the listener 
     * at each key
     * @param bucket the bucket to iterate
     * @param listener the listener
     */
public void paginate(String bucket, BucketKeyListener listener) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    for (S3ObjectSummary summary : list.getObjectSummaries()) {
        if (listener != null)
            listener.onKey(s3, bucket, summary.getKey());
    }
    while (list.isTruncated()) {
        list = s3.listNextBatchOfObjects(list);
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            if (listener != null)
                listener.onKey(s3, bucket, summary.getKey());
        }
    }
}

Example 67 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project deeplearning4j by deeplearning4j.

the class S3Downloader method listObjects.

/**
     * Simple way of retrieving the listings for a bucket
     * @param bucket the bucket to retrieve listings for
     * @return the object listing for this bucket
     */
public ObjectListing listObjects(String bucket) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    return list;
}

Example 68 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project GNS by MobilityFirst.

the class AWSStatusCheck method main.

/**
   *
   * @param args
   * @throws Exception
   */
public static void main(String[] args) throws Exception {
    init();
    /*
     * Amazon EC2
     */
    for (String endpoint : endpoints) {
        try {
            ec2.setEndpoint(endpoint);
            System.out.println("**** Endpoint: " + endpoint);
            DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
            System.out.println("You have access to " + availabilityZonesResult.getAvailabilityZones().size() + " Availability Zones.");
            for (AvailabilityZone zone : availabilityZonesResult.getAvailabilityZones()) {
                System.out.println(zone.getZoneName());
            }
            DescribeInstancesResult describeInstancesRequest = ec2.describeInstances();
            List<Reservation> reservations = describeInstancesRequest.getReservations();
            Set<Instance> instances = new HashSet<Instance>();
            System.out.println("Instances: ");
            for (Reservation reservation : reservations) {
                for (Instance instance : reservation.getInstances()) {
                    instances.add(instance);
                    System.out.println(instance.getPublicDnsName() + " is " + instance.getState().getName());
                }
            }
            System.out.println("Security groups: ");
            DescribeSecurityGroupsResult describeSecurityGroupsResult = ec2.describeSecurityGroups();
            for (SecurityGroup securityGroup : describeSecurityGroupsResult.getSecurityGroups()) {
                System.out.println(securityGroup.getGroupName());
            }
        //System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon SimpleDB
       *
       */
        try {
            ListDomainsRequest sdbRequest = new ListDomainsRequest().withMaxNumberOfDomains(100);
            ListDomainsResult sdbResult = sdb.listDomains(sdbRequest);
            int totalItems = 0;
            for (String domainName : sdbResult.getDomainNames()) {
                DomainMetadataRequest metadataRequest = new DomainMetadataRequest().withDomainName(domainName);
                DomainMetadataResult domainMetadata = sdb.domainMetadata(metadataRequest);
                totalItems += domainMetadata.getItemCount();
            }
            System.out.println("You have " + sdbResult.getDomainNames().size() + " Amazon SimpleDB domain(s)" + "containing a total of " + totalItems + " items.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon S3
       *.
       */
        try {
            List<Bucket> buckets = s3.listBuckets();
            long totalSize = 0;
            int totalItems = 0;
            for (Bucket bucket : buckets) {
                /*
           * In order to save bandwidth, an S3 object listing does not
           * contain every object in the bucket; after a certain point the
           * S3ObjectListing is truncated, and further pages must be
           * obtained with the AmazonS3Client.listNextBatchOfObjects()
           * method.
           */
                ObjectListing objects = s3.listObjects(bucket.getName());
                do {
                    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                        totalSize += objectSummary.getSize();
                        totalItems++;
                    }
                    objects = s3.listNextBatchOfObjects(objects);
                } while (objects.isTruncated());
            }
            System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s), " + "containing " + totalItems + " objects with a total size of " + totalSize + " bytes.");
        } catch (AmazonServiceException ase) {
            /*
         * AmazonServiceExceptions represent an error response from an AWS
         * services, i.e. your request made it to AWS, but the AWS service
         * either found it invalid or encountered an error trying to execute
         * it.
         */
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
        } catch (AmazonClientException ace) {
            /*
         * AmazonClientExceptions represent an error that occurred inside
         * the client on the local host, either while trying to send the
         * request to AWS or interpret the response. For example, if no
         * network connection is available, the client won't be able to
         * connect to AWS to execute a request and will throw an
         * AmazonClientException.
         */
            System.out.println("Error Message: " + ace.getMessage());
        }
    }
}

Example 69 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project camel by apache.

the class S3Consumer method poll.

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }
        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }
        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}

Example 70 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project camel by apache.

the class AmazonS3ClientMock method listObjects.

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex;
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }
    return objectListing;
}