Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 76 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project hadoop by apache.

the class TestS3AGetFileStatus method testImplicitDirectory.

@Test
public void testImplicitDirectory() throws Exception {
    Path path = new Path("/dir");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.singletonList("dir/"));
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
}

Example 77 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project hadoop by apache.

the class TestS3AGetFileStatus method testRoot.

@Test
public void testRoot() throws Exception {
    Path path = new Path("/");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.<String>emptyList());
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
    assertTrue(stat.getPath().isRoot());
}

Example 78 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project hadoop by apache.

the class S3AFileSystem method innerDelete.

/**
   * Delete an object. See {@link #delete(Path, boolean)}.
   *
   * @param status fileStatus object
   * @param recursive if path is a directory and set to
   * true, the directory is deleted else throws an exception. In
   * case of a file the recursive can be set to either true or false.
   * @return  true if delete is successful else false.
   * @throws IOException due to inability to delete a directory or file.
   * @throws AmazonClientException on failures inside the AWS SDK
   */
private boolean innerDelete(S3AFileStatus status, boolean recursive) throws IOException, AmazonClientException {
    Path f = status.getPath();
    LOG.debug("Delete path {} - recursive {}", f, recursive);
    String key = pathToKey(f);
    if (status.isDirectory()) {
        LOG.debug("delete: Path is a directory: {}", f);
        if (!key.endsWith("/")) {
            key = key + "/";
        }
        if (key.equals("/")) {
            return rejectRootDirectoryDelete(status, recursive);
        }
        if (!recursive && !status.isEmptyDirectory()) {
            throw new PathIsNotEmptyDirectoryException(f.toString());
        }
        if (status.isEmptyDirectory()) {
            LOG.debug("Deleting fake empty directory {}", key);
            deleteObject(key);
            instrumentation.directoryDeleted();
        } else {
            LOG.debug("Getting objects for directory prefix {} to delete", key);
            ListObjectsRequest request = createListObjectsRequest(key, null);
            ObjectListing objects = listObjects(request);
            List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(objects.getObjectSummaries().size());
            while (true) {
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                    LOG.debug("Got object to delete {}", summary.getKey());
                    if (keys.size() == MAX_ENTRIES_TO_DELETE) {
                        removeKeys(keys, true, false);
                    }
                }
                if (objects.isTruncated()) {
                    objects = continueListObjects(objects);
                } else {
                    if (!keys.isEmpty()) {
                        removeKeys(keys, false, false);
                    }
                    break;
                }
            }
        }
    } else {
        LOG.debug("delete: Path is a file");
        instrumentation.fileDeleted(1);
        deleteObject(key);
    }
    Path parent = f.getParent();
    if (parent != null) {
        createFakeDirectoryIfNecessary(parent);
    }
    return true;
}

Example 79 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project nifi by apache.

the class AbstractS3IT method oneTimeTearDown.

@AfterClass
public static void oneTimeTearDown() {
    // Empty the bucket before deleting it.
    try {
        ObjectListing objectListing = client.listObjects(BUCKET_NAME);
        while (true) {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                client.deleteObject(BUCKET_NAME, objectSummary.getKey());
            }
            if (objectListing.isTruncated()) {
                objectListing = client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
        client.deleteBucket(dbr);
    } catch (final AmazonS3Exception e) {
        System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
    }
    if (client.doesBucketExist(BUCKET_NAME)) {
        Assert.fail("Incomplete teardown, subsequent tests might fail");
    }
}

Example 80 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project nifi by apache.

the class TestListS3 method testListVersion2.

@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");
    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);
    runner.run();
    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}