Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 21 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testListObjectsAllObjectsMultiplePagesModTime.

@Test
public void testListObjectsAllObjectsMultiplePagesModTime() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList1 = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectSummary.setLastModified(new Date(100));
    objectList1.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectSummary.setLastModified(new Date(100));
    objectList1.add(objectSummary);
    ArrayList<S3ObjectSummary> objectList2 = new ArrayList<>();
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("cd");
    objectSummary.setLastModified(new Date(100));
    objectList2.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("cd.docker");
    objectSummary.setLastModified(new Date(200));
    objectList2.add(objectSummary);
    ArrayList<S3ObjectSummary> objectList3 = new ArrayList<>();
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("platforms");
    objectSummary.setLastModified(new Date(200));
    objectList3.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("platforms.mh2");
    objectSummary.setLastModified(new Date(200));
    objectList3.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList1).thenReturn(objectList2).thenReturn(objectList3);
    when(objectListing.isTruncated()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
    ArrayList<String> domains = new ArrayList<>();
    store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime());
    assertEquals(domains.size(), 3);
    assertTrue(domains.contains("cd.docker"));
    assertTrue(domains.contains("platforms"));
    assertTrue(domains.contains("platforms.mh2"));
}

Example 22 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetUpdatedSignedDomainsWithChange.

@Test
public void testGetUpdatedSignedDomainsWithChange() throws FileNotFoundException {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectSummary.setLastModified(new Date(100));
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectSummary.setLastModified(new Date(200));
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    InputStream is = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null);
    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is);
    when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
    when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas.athenz")).thenReturn(object);
    // set the last modification time to return one of the domains
    store.lastModTime = (new Date(150)).getTime();
    StringBuilder lastModTimeBuffer = new StringBuilder(512);
    SignedDomains signedDomains = store.getUpdatedSignedDomains(lastModTimeBuffer);
    assertTrue(lastModTimeBuffer.length() > 0);
    List<SignedDomain> domainList = signedDomains.getDomains();
    assertEquals(domainList.size(), 1);
    DomainData domainData = domainList.get(0).getDomain();
    assertNotNull(domainData);
    assertEquals(domainData.getName(), "iaas");
}

Example 23 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testListObjectsAllObjectsNoPageModTime.

@Test
public void testListObjectsAllObjectsNoPageModTime() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectSummary.setLastModified(new Date(100));
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectSummary.setLastModified(new Date(200));
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    ArrayList<String> domains = new ArrayList<>();
    store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime());
    assertEquals(domains.size(), 1);
    assertTrue(domains.contains("iaas.athenz"));
}

Example 24 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testListObjectsAllObjectsMultiplePages.

@Test
public void testListObjectsAllObjectsMultiplePages() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList1 = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectList1.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectList1.add(objectSummary);
    ArrayList<S3ObjectSummary> objectList2 = new ArrayList<>();
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("cd");
    objectList2.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("cd.docker");
    objectList2.add(objectSummary);
    ArrayList<S3ObjectSummary> objectList3 = new ArrayList<>();
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("platforms");
    objectList3.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("platforms.mh2");
    objectList3.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList1).thenReturn(objectList2).thenReturn(objectList3);
    when(objectListing.isTruncated()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
    ArrayList<String> domains = new ArrayList<>();
    store.listObjects(store.awsS3Client, domains, 0);
    assertEquals(domains.size(), 6);
    assertTrue(domains.contains("iaas"));
    assertTrue(domains.contains("iaas.athenz"));
    assertTrue(domains.contains("cd"));
    assertTrue(domains.contains("cd.docker"));
    assertTrue(domains.contains("platforms"));
    assertTrue(domains.contains("platforms.mh2"));
}

Example 25 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStore method listObjects.

/**
 * list the objects in the zts bucket. If the mod time is specified as 0
 * then we want to list all objects otherwise, we only list objects
 * that are newer than the specified timestamp
 * @param s3 AWS S3 client object
 * @param domains collection to be updated to include domain names
 * @param modTime only include domains newer than this timestamp
 */
void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}", s3BucketName, modTime);
    }
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName));
    String objectName = null;
    while (objectListing != null) {
        // process each entry in our result set and add the domain
        // name to our return list
        final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
        boolean listTruncated = objectListing.isTruncated();
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}", objectSummaries.size(), listTruncated);
        }
        for (S3ObjectSummary objectSummary : objectSummaries) {
            if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) {
                continue;
            }
            // for now skip any folders/objects that start with '.'
            objectName = objectSummary.getKey();
            if (objectName.charAt(0) == '.') {
                continue;
            }
            domains.add(objectName);
        }
        if (!listTruncated) {
            break;
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    }
}