Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 41 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class TestS3AGetFileStatus method testImplicitDirectory.

@Test
public void testImplicitDirectory() throws Exception {
    Path path = new Path("/dir");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.singletonList("dir/"));
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
}

Example 42 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class TestS3AGetFileStatus method testRoot.

@Test
public void testRoot() throws Exception {
    Path path = new Path("/");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.<String>emptyList());
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
    assertTrue(stat.getPath().isRoot());
}

Example 43 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project ice by Netflix.

the class AwsUtils method listAllObjects.

/**
     * List all object summary with given prefix in the s3 bucket.
     * @param bucket
     * @param prefix
     * @return
     */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;
    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig);
        }
        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());
        } while (page.isTruncated());
        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

Example 44 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project GNS by MobilityFirst.

the class AWSStatusCheck method main.

/**
   *
   * @param args
   * @throws Exception
   */
public static void main(String[] args) throws Exception {
    init();
    /*
     * Amazon EC2
     */
    for (String endpoint : endpoints) {
        try {
            ec2.setEndpoint(endpoint);
            System.out.println("**** Endpoint: " + endpoint);
            DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
            System.out.println("You have access to " + availabilityZonesResult.getAvailabilityZones().size() + " Availability Zones.");
            for (AvailabilityZone zone : availabilityZonesResult.getAvailabilityZones()) {
                System.out.println(zone.getZoneName());
            }
            DescribeInstancesResult describeInstancesRequest = ec2.describeInstances();
            List<Reservation> reservations = describeInstancesRequest.getReservations();
            Set<Instance> instances = new HashSet<Instance>();
            System.out.println("Instances: ");
            for (Reservation reservation : reservations) {
                for (Instance instance : reservation.getInstances()) {
                    instances.add(instance);
                    System.out.println(instance.getPublicDnsName() + " is " + instance.getState().getName());
                }
            }
            System.out.println("Security groups: ");
            DescribeSecurityGroupsResult describeSecurityGroupsResult = ec2.describeSecurityGroups();
            for (SecurityGroup securityGroup : describeSecurityGroupsResult.getSecurityGroups()) {
                System.out.println(securityGroup.getGroupName());
            }
        //System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon SimpleDB
       *
       */
        try {
            ListDomainsRequest sdbRequest = new ListDomainsRequest().withMaxNumberOfDomains(100);
            ListDomainsResult sdbResult = sdb.listDomains(sdbRequest);
            int totalItems = 0;
            for (String domainName : sdbResult.getDomainNames()) {
                DomainMetadataRequest metadataRequest = new DomainMetadataRequest().withDomainName(domainName);
                DomainMetadataResult domainMetadata = sdb.domainMetadata(metadataRequest);
                totalItems += domainMetadata.getItemCount();
            }
            System.out.println("You have " + sdbResult.getDomainNames().size() + " Amazon SimpleDB domain(s)" + "containing a total of " + totalItems + " items.");
        } catch (AmazonServiceException ase) {
            System.out.println("Caught Exception: " + ase.getMessage());
            System.out.println("Reponse Status Code: " + ase.getStatusCode());
            System.out.println("Error Code: " + ase.getErrorCode());
            System.out.println("Request ID: " + ase.getRequestId());
        }
        /*
       * Amazon S3
       *.
       */
        try {
            List<Bucket> buckets = s3.listBuckets();
            long totalSize = 0;
            int totalItems = 0;
            for (Bucket bucket : buckets) {
                /*
           * In order to save bandwidth, an S3 object listing does not
           * contain every object in the bucket; after a certain point the
           * S3ObjectListing is truncated, and further pages must be
           * obtained with the AmazonS3Client.listNextBatchOfObjects()
           * method.
           */
                ObjectListing objects = s3.listObjects(bucket.getName());
                do {
                    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                        totalSize += objectSummary.getSize();
                        totalItems++;
                    }
                    objects = s3.listNextBatchOfObjects(objects);
                } while (objects.isTruncated());
            }
            System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s), " + "containing " + totalItems + " objects with a total size of " + totalSize + " bytes.");
        } catch (AmazonServiceException ase) {
            /*
         * AmazonServiceExceptions represent an error response from an AWS
         * services, i.e. your request made it to AWS, but the AWS service
         * either found it invalid or encountered an error trying to execute
         * it.
         */
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
        } catch (AmazonClientException ace) {
            /*
         * AmazonClientExceptions represent an error that occurred inside
         * the client on the local host, either while trying to send the
         * request to AWS or interpret the response. For example, if no
         * network connection is available, the client won't be able to
         * connect to AWS to execute a request and will throw an
         * AmazonClientException.
         */
            System.out.println("Error Message: " + ace.getMessage());
        }
    }
}

Example 45 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project cloudstack by apache.

the class NfsSecondaryStorageResource method s3ListTemplate.

Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
    String bucket = s3.getBucketName();
    // List the objects in the source directory on S3
    final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
    if (objectSummaries == null) {
        return null;
    }
    Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
    for (S3ObjectSummary objectSummary : objectSummaries) {
        String key = objectSummary.getKey();
        // String installPath = StringUtils.substringBeforeLast(key,
        // S3Utils.SEPARATOR);
        String uniqueName = determineS3TemplateNameFromKey(key);
        // TODO: isPublic value, where to get?
        TemplateProp tInfo = new TemplateProp(uniqueName, key, objectSummary.getSize(), objectSummary.getSize(), true, false);
        tmpltInfos.put(uniqueName, tInfo);
    }
    return tmpltInfos;
}