Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 81 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project nifi by apache.

the class TestListS3 method testList.

@Test
public void testList() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);
    runner.run();
    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 82 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetUpdatedSignedDomainsNoChanges.

@Test
public void testGetUpdatedSignedDomainsNoChanges() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectSummary.setLastModified(new Date(100));
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectSummary.setLastModified(new Date(200));
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    // set the last modification time to not return any of the domains
    store.lastModTime = (new Date(250)).getTime();
    StringBuilder lastModTimeBuffer = new StringBuilder(512);
    SignedDomains signedDomains = store.getUpdatedSignedDomains(lastModTimeBuffer);
    assertTrue(lastModTimeBuffer.length() > 0);
    assertEquals(signedDomains.getDomains().size(), 0);
}

Example 83 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testListObjectsAllObjectsNoPage.

@Test
public void testListObjectsAllObjectsNoPage() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    ArrayList<String> domains = new ArrayList<>();
    store.listObjects(store.awsS3Client, domains, 0);
    assertEquals(domains.size(), 2);
    assertTrue(domains.contains("iaas"));
    assertTrue(domains.contains("iaas.athenz"));
}

Example 84 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetLocalDomains.

@Test
public void testGetLocalDomains() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    // verify that our last mod time is 0 before the call
    assertEquals(store.lastModTime, 0);
    // retrieve the list of domains
    List<String> domains = store.getLocalDomainList();
    assertEquals(domains.size(), 2);
    assertTrue(domains.contains("iaas"));
    assertTrue(domains.contains("iaas.athenz"));
    // also verify that last mod time is updated
    assertTrue(store.lastModTime > 0);
}

Example 85 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project athenz by yahoo.

the class S3ChangeLogStoreTest method testGetServerDomains.

@Test
public void testGetServerDomains() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    ArrayList<S3ObjectSummary> objectList = new ArrayList<>();
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas");
    objectList.add(objectSummary);
    objectSummary = new S3ObjectSummary();
    objectSummary.setKey("iaas.athenz");
    objectList.add(objectSummary);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(objectList);
    when(objectListing.isTruncated()).thenReturn(false);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    // verify that our last mod time is 0 before the call
    assertEquals(store.lastModTime, 0);
    // retrieve the list of domains
    Set<String> domains = store.getServerDomainList();
    assertEquals(domains.size(), 2);
    assertTrue(domains.contains("iaas"));
    assertTrue(domains.contains("iaas.athenz"));
    // also verify that last mod time is not updated
    assertEquals(store.lastModTime, 0);
}