Examples with ListObjectsRequest software.amazon.awssdk.services.s3.model.ListObjectsRequest used on opensource projects

Example 41 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project herd by FINRAOS.

the class S3DaoImpl method listDirectory.

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;
        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L)) {
                    s3ObjectSummaries.add(objectSummary);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return s3ObjectSummaries;
}

Example 42 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project herd by FINRAOS.

the class S3DaoTest method testListDirectoryAssertTruncatedResult.

@Test
public void testListDirectoryAssertTruncatedResult() {
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    try {
        String s3BucketName = "s3BucketName";
        String s3KeyPrefix = "s3KeyPrefix";
        String expectedFilePath = "key";
        long expectedFileSize = 1024l;
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);
        boolean ignoreZeroByteDirectoryMarkers = true;
        when(mockS3Operations.listObjects(any(), any())).then(new Answer<ObjectListing>() {

            @Override
            public ObjectListing answer(InvocationOnMock invocation) throws Throwable {
                ListObjectsRequest listObjectsRequest = invocation.getArgument(0);
                String marker = listObjectsRequest.getMarker();
                ObjectListing objectListing = new ObjectListing();
                if (marker == null) {
                    objectListing.setNextMarker("marker");
                    objectListing.setTruncated(true);
                } else {
                    assertEquals("marker", marker);
                    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                    s3ObjectSummary.setKey(expectedFilePath);
                    s3ObjectSummary.setSize(expectedFileSize);
                    objectListing.getObjectSummaries().add(s3ObjectSummary);
                }
                return objectListing;
            }
        });
        List<S3ObjectSummary> s3ObjectSummaries = s3Dao.listDirectory(s3FileTransferRequestParamsDto, ignoreZeroByteDirectoryMarkers);
        assertEquals(1, s3ObjectSummaries.size());
        assertEquals(expectedFilePath, s3ObjectSummaries.get(0).getKey());
        assertEquals(expectedFileSize, s3ObjectSummaries.get(0).getSize());
    } finally {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

Example 43 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project aws-doc-sdk-examples by awsdocs.

the class S3Service method listBucketObjects.

// Returns the names of all images in the given bucket.
public List<String> listBucketObjects(String bucketName) {
    S3Client s3 = getClient();
    String keyName;
    List<String> keys = new ArrayList<>();
    try {
        ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(bucketName).build();
        ListObjectsResponse res = s3.listObjects(listObjects);
        List<S3Object> objects = res.contents();
        for (S3Object myValue : objects) {
            keyName = myValue.key();
            keys.add(keyName);
        }
        return keys;
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
    return null;
}

Example 44 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project aws-doc-sdk-examples by awsdocs.

the class VPCS3Example method listBucketObjects.

// snippet-start:[s3.java2.vpc.example.main]
public static void listBucketObjects(S3Client s3, String bucketName) {
    try {
        ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(bucketName).build();
        ListObjectsResponse res = s3.listObjects(listObjects);
        List<S3Object> objects = res.contents();
        for (S3Object s3Object : objects) {
            System.out.print("\n The name of the key is " + s3Object.key());
            System.out.print("\n The object is " + convertBToKb(s3Object.size()) + " KBs");
            System.out.print("\n The owner is " + s3Object.owner());
        }
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}

Example 45 with ListObjectsRequest

use of software.amazon.awssdk.services.s3.model.ListObjectsRequest in project stocator by SparkTC.

the class COSAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path f, String msg) throws IOException, FileNotFoundException {
    Path path = f;
    boolean originalTempTarget = false;
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        if (!path.toString().contains(HADOOP_PART)) {
            LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
            throw new FileNotFoundException("Not found " + path.toString());
        }
        path = stocatorPath.modifyPathToFinalDestination(f);
        LOG.debug("getFileStatus on temp object {}. Modify to final name {}", f.toString(), path.toString());
        originalTempTarget = true;
    }
    FileStatus res = null;
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    String key = pathToKey(path);
    LOG.debug("getFileStatus: on original key {}", key);
    FileStatus fileStatus = null;
    try {
        fileStatus = getFileStatusKeyBased(key, path);
    } catch (AmazonS3Exception e) {
        LOG.warn("file status {} returned {}", key, e.getStatusCode());
        if (e.getStatusCode() != 404) {
            LOG.warn("Throw IOException for {}. Most likely authentication failed", key);
            throw new IOException(e);
        } else if (originalTempTarget && e.getStatusCode() == 404) {
            throw new FileNotFoundException("Not found " + f.toString());
        }
    }
    if (fileStatus != null) {
        LOG.trace("getFileStatus(completed) {}", path);
        memoryCache.putFileStatus(path.toString(), fileStatus);
        return fileStatus;
    }
    // probably not needed this call
    if (!key.endsWith("/")) {
        String newKey = key + "/";
        try {
            LOG.debug("getFileStatus: original key not found. Alternative key {}", newKey);
            fileStatus = getFileStatusKeyBased(newKey, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        } else {
            // if here: both key and key/ returned not found.
            // trying to see if pseudo directory of the form
            // a/b/key/d/e (a/b/key/ doesn't exists by itself)
            // perform listing on the key
            LOG.debug("getFileStatus: Modifined key {} not found. Trying to list", key);
            key = maybeAddTrailingSlash(key);
            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(mBucket);
            request.setPrefix(key);
            request.withEncodingType("url");
            request.setDelimiter("/");
            request.setMaxKeys(1);
            ObjectListing objects = mClient.listObjects(request);
            if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            } else if (key.isEmpty()) {
                LOG.trace("Found root directory");
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            }
        }
    }
    LOG.debug("Not found {}. Throw FNF exception", path.toString());
    throw new FileNotFoundException("Not found " + path.toString());
}