Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 36 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project YCSB by brianfrankcooper.

the class S3Client method scanFromStorage.

/**
  * Perform an emulation of a database scan operation on a S3 bucket.
  *
  * @param bucket
  *            The name of the bucket
  * @param startkey
  *            The file key of the first file to read.
  * @param recordcount
  *            The number of files to read
  * @param fields
  *            The list of fields to read, or null for all of them
  * @param result
  *            A Vector of HashMaps, where each HashMap is a set field/value
  *            pairs for one file
  *
  */
protected Status scanFromStorage(String bucket, String startkey, int recordcount, Vector<HashMap<String, ByteIterator>> result, SSECustomerKey ssecLocal) {
    int counter = 0;
    ObjectListing listing = s3Client.listObjects(bucket);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    List<String> keyList = new ArrayList();
    int startkeyNumber = 0;
    int numberOfIteration = 0;
    // getting the list of files in the bucket
    while (listing.isTruncated()) {
        listing = s3Client.listNextBatchOfObjects(listing);
        summaries.addAll(listing.getObjectSummaries());
    }
    for (S3ObjectSummary summary : summaries) {
        String summaryKey = summary.getKey();
        keyList.add(summaryKey);
    }
    // Sorting the list of files in Alphabetical order
    // sorting the list
    Collections.sort(keyList);
    // Getting the position of the startingfile for the scan
    for (String key : keyList) {
        if (key.equals(startkey)) {
            startkeyNumber = counter;
        } else {
            counter = counter + 1;
        }
    }
    // if not using the total number of Files
    if (recordcount < keyList.size()) {
        numberOfIteration = recordcount;
    } else {
        numberOfIteration = keyList.size();
    }
    // of the Files or Till the recordcount number
    for (int i = startkeyNumber; i < numberOfIteration; i++) {
        HashMap<String, ByteIterator> resultTemp = new HashMap<String, ByteIterator>();
        readFromStorage(bucket, keyList.get(i), resultTemp, ssecLocal);
        result.add(resultTemp);
    }
    return Status.OK;
}

Example 37 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project crate by crate.

the class S3FileInputTest method testListListUrlsWhenEmptyKeysIsListed.

@Test
public void testListListUrlsWhenEmptyKeysIsListed() throws Exception {
    S3ObjectSummary path = new S3ObjectSummary();
    path.setBucketName(BUCKET_NAME);
    path.setKey("prefix/");
    listObjectSummaries = objectSummaries();
    listObjectSummaries.add(path);
    when(objectListing.getObjectSummaries()).thenReturn(listObjectSummaries);
    List<URI> uris = s3FileInput.listUris(uri, uriPredicate);
    assertThat(uris.size(), is(2));
    assertThat(uris.get(0).toString(), is("s3://fakeBucket/prefix/test1.json.gz"));
    assertThat(uris.get(1).toString(), is("s3://fakeBucket/prefix/test2.json.gz"));
}

Example 38 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project deeplearning4j by deeplearning4j.

the class S3Downloader method paginate.

/**
     * Paginates through a bucket's keys invoking the listener 
     * at each key
     * @param bucket the bucket to iterate
     * @param listener the listener
     */
public void paginate(String bucket, BucketKeyListener listener) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    for (S3ObjectSummary summary : list.getObjectSummaries()) {
        if (listener != null)
            listener.onKey(s3, bucket, summary.getKey());
    }
    while (list.isTruncated()) {
        list = s3.listNextBatchOfObjects(list);
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            if (listener != null)
                listener.onKey(s3, bucket, summary.getKey());
        }
    }
}

Example 39 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project ice by Netflix.

the class AwsUtils method listAllObjects.

/**
     * List all object summary with given prefix in the s3 bucket.
     * @param bucket
     * @param prefix
     * @return
     */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;
    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig);
        }
        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());
        } while (page.isTruncated());
        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

Example 40 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project GNS by MobilityFirst.

the class AWSStatusCheck method main.

/**
   *
   * @param args
   * @throws Exception
   */
public static void main(String[] args) throws Exception {
    init();
    /*
     * Amazon EC2
     */
    for (String endpoint : endpoints) {
        try {
            ec2.setEndpoint(endpoint);
            System.out.println("**** Endpoint: " + endpoint);
            DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
            System.out.println("You have access to " + availabilityZonesResult.getAvailabilityZones().size() + " Availability Zones.");
            for (AvailabilityZone zone : availabilityZonesResult.getAvailabilityZones()) {
                System.out.println(zone.getZoneName());
            }
            DescribeInstancesResult describeInstancesRequest = ec2.describeInstances();
            List<Reservation> reservations = describeInstancesRequest.getReservations();
            Set<Instance> instances = new HashSet<Instance>();
            System.out.println("Instances: ");
            for (Reservation reservation : reservations) {
                for (Instance instance : reservation.getInstances()) {
                    instances.add(instance);
                    System.out.println(instance.getPublicDnsName() + " is " + instance.getState().getName());
                }
            }
            System.out.println("Security groups: ");
            DescribeSecurityGroupsResult describeSecurityGroupsResult = ec2.describeSecurityGroups();
            for (SecurityGroup securityGroup : describeSecurityGroupsResult.getSecurityGroups()) {
                System.out.println(securityGroup.getGroupName());
            }
        //System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon SimpleDB
       *
       */
        try {
            ListDomainsRequest sdbRequest = new ListDomainsRequest().withMaxNumberOfDomains(100);
            ListDomainsResult sdbResult = sdb.listDomains(sdbRequest);
            int totalItems = 0;
            for (String domainName : sdbResult.getDomainNames()) {
                DomainMetadataRequest metadataRequest = new DomainMetadataRequest().withDomainName(domainName);
                DomainMetadataResult domainMetadata = sdb.domainMetadata(metadataRequest);
                totalItems += domainMetadata.getItemCount();
            }
            System.out.println("You have " + sdbResult.getDomainNames().size() + " Amazon SimpleDB domain(s)" + "containing a total of " + totalItems + " items.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon S3
       *.
       */
        try {
            List<Bucket> buckets = s3.listBuckets();
            long totalSize = 0;
            int totalItems = 0;
            for (Bucket bucket : buckets) {
                /*
           * In order to save bandwidth, an S3 object listing does not
           * contain every object in the bucket; after a certain point the
           * S3ObjectListing is truncated, and further pages must be
           * obtained with the AmazonS3Client.listNextBatchOfObjects()
           * method.
           */
                ObjectListing objects = s3.listObjects(bucket.getName());
                do {
                    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                        totalSize += objectSummary.getSize();
                        totalItems++;
                    }
                    objects = s3.listNextBatchOfObjects(objects);
                } while (objects.isTruncated());
            }
            System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s), " + "containing " + totalItems + " objects with a total size of " + totalSize + " bytes.");
        } catch (AmazonServiceException ase) {
            /*
         * AmazonServiceExceptions represent an error response from an AWS
         * services, i.e. your request made it to AWS, but the AWS service
         * either found it invalid or encountered an error trying to execute
         * it.
         */
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
        } catch (AmazonClientException ace) {
            /*
         * AmazonClientExceptions represent an error that occurred inside
         * the client on the local host, either while trying to send the
         * request to AWS or interpret the response. For example, if no
         * network connection is available, the client won't be able to
         * connect to AWS to execute a request and will throw an
         * AmazonClientException.
         */
            System.out.println("Error Message: " + ace.getMessage());
        }
    }
}