Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 56 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class S3AFileSystem method getFileStatus.

/**
   * Return a file status object that represents the path.
   * @param f The path we want information from
   * @return a FileStatus object
   * @throws java.io.FileNotFoundException when the path does not exist;
   * @throws IOException on other problems.
   */
public S3AFileStatus getFileStatus(final Path f) throws IOException {
    incrementStatistic(INVOCATION_GET_FILE_STATUS);
    final Path path = qualify(f);
    String key = pathToKey(path);
    LOG.debug("Getting path status for {}  ({})", path, key);
    if (!key.isEmpty()) {
        try {
            ObjectMetadata meta = getObjectMetadata(key);
            if (objectRepresentsDirectory(key, meta.getContentLength())) {
                LOG.debug("Found exact file: fake directory");
                return new S3AFileStatus(true, path, username);
            } else {
                LOG.debug("Found exact file: normal file");
                return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
            }
        } catch (AmazonServiceException e) {
            if (e.getStatusCode() != 404) {
                throw translateException("getFileStatus", path, e);
            }
        } catch (AmazonClientException e) {
            throw translateException("getFileStatus", path, e);
        }
        // Necessary?
        if (!key.endsWith("/")) {
            String newKey = key + "/";
            try {
                ObjectMetadata meta = getObjectMetadata(newKey);
                if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
                    LOG.debug("Found file (with /): fake directory");
                    return new S3AFileStatus(true, path, username);
                } else {
                    LOG.warn("Found file (with /): real file? should not happen: {}", key);
                    return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), path, getDefaultBlockSize(path), username);
                }
            } catch (AmazonServiceException e) {
                if (e.getStatusCode() != 404) {
                    throw translateException("getFileStatus", newKey, e);
                }
            } catch (AmazonClientException e) {
                throw translateException("getFileStatus", newKey, e);
            }
        }
    }
    try {
        key = maybeAddTrailingSlash(key);
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(1);
        ObjectListing objects = listObjects(request);
        if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Found path as directory (with /): {}/{}", objects.getCommonPrefixes().size(), objects.getObjectSummaries().size());
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    LOG.debug("Summary: {} {}", summary.getKey(), summary.getSize());
                }
                for (String prefix : objects.getCommonPrefixes()) {
                    LOG.debug("Prefix: {}", prefix);
                }
            }
            return new S3AFileStatus(false, path, username);
        } else if (key.isEmpty()) {
            LOG.debug("Found root directory");
            return new S3AFileStatus(true, path, username);
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404) {
            throw translateException("getFileStatus", key, e);
        }
    } catch (AmazonClientException e) {
        throw translateException("getFileStatus", key, e);
    }
    LOG.debug("Not Found: {}", path);
    throw new FileNotFoundException("No such file or directory: " + path);
}

Example 57 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class TestS3AGetFileStatus method testNotFound.

@Test
public void testNotFound() throws Exception {
    Path path = new Path("/dir");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.<String>emptyList());
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    exception.expect(FileNotFoundException.class);
    fs.getFileStatus(path);
}

Example 58 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class TestS3AGetFileStatus method testImplicitDirectory.

@Test
public void testImplicitDirectory() throws Exception {
    Path path = new Path("/dir");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.singletonList("dir/"));
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
}

Example 59 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class TestS3AGetFileStatus method testRoot.

@Test
public void testRoot() throws Exception {
    Path path = new Path("/");
    String key = path.toUri().getPath().substring(1);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))).thenThrow(NOT_FOUND);
    when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key + "/")))).thenThrow(NOT_FOUND);
    ObjectListing objects = mock(ObjectListing.class);
    when(objects.getCommonPrefixes()).thenReturn(Collections.<String>emptyList());
    when(objects.getObjectSummaries()).thenReturn(Collections.<S3ObjectSummary>emptyList());
    when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
    FileStatus stat = fs.getFileStatus(path);
    assertNotNull(stat);
    assertEquals(fs.makeQualified(path), stat.getPath());
    assertTrue(stat.isDirectory());
    assertTrue(stat.getPath().isRoot());
}

Example 60 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project hadoop by apache.

the class S3AFileSystem method innerDelete.

/**
   * Delete an object. See {@link #delete(Path, boolean)}.
   *
   * @param status fileStatus object
   * @param recursive if path is a directory and set to
   * true, the directory is deleted else throws an exception. In
   * case of a file the recursive can be set to either true or false.
   * @return  true if delete is successful else false.
   * @throws IOException due to inability to delete a directory or file.
   * @throws AmazonClientException on failures inside the AWS SDK
   */
private boolean innerDelete(S3AFileStatus status, boolean recursive) throws IOException, AmazonClientException {
    Path f = status.getPath();
    LOG.debug("Delete path {} - recursive {}", f, recursive);
    String key = pathToKey(f);
    if (status.isDirectory()) {
        LOG.debug("delete: Path is a directory: {}", f);
        if (!key.endsWith("/")) {
            key = key + "/";
        }
        if (key.equals("/")) {
            return rejectRootDirectoryDelete(status, recursive);
        }
        if (!recursive && !status.isEmptyDirectory()) {
            throw new PathIsNotEmptyDirectoryException(f.toString());
        }
        if (status.isEmptyDirectory()) {
            LOG.debug("Deleting fake empty directory {}", key);
            deleteObject(key);
            instrumentation.directoryDeleted();
        } else {
            LOG.debug("Getting objects for directory prefix {} to delete", key);
            ListObjectsRequest request = createListObjectsRequest(key, null);
            ObjectListing objects = listObjects(request);
            List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(objects.getObjectSummaries().size());
            while (true) {
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                    LOG.debug("Got object to delete {}", summary.getKey());
                    if (keys.size() == MAX_ENTRIES_TO_DELETE) {
                        removeKeys(keys, true, false);
                    }
                }
                if (objects.isTruncated()) {
                    objects = continueListObjects(objects);
                } else {
                    if (!keys.isEmpty()) {
                        removeKeys(keys, false, false);
                    }
                    break;
                }
            }
        }
    } else {
        LOG.debug("delete: Path is a file");
        instrumentation.fileDeleted(1);
        deleteObject(key);
    }
    Path parent = f.getParent();
    if (parent != null) {
        createFakeDirectoryIfNecessary(parent);
    }
    return true;
}