Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 31 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project spring-integration-aws by spring-projects.

the class S3Session method list.

@Override
public S3ObjectSummary[] list(String path) throws IOException {
    String[] bucketPrefix = splitPathToBucketAndKey(path, false);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketPrefix[0]);
    if (bucketPrefix.length > 1) {
        listObjectsRequest.setPrefix(bucketPrefix[1]);
    }
    /*
		For listing objects, Amazon S3 returns up to 1,000 keys in the response.
		If you have more than 1,000 keys in your bucket, the response will be truncated.
		You should always check for if the response is truncated.
		*/
    ObjectListing objectListing;
    List<S3ObjectSummary> objectSummaries = new ArrayList<>();
    do {
        objectListing = this.amazonS3.listObjects(listObjectsRequest);
        objectSummaries.addAll(objectListing.getObjectSummaries());
        listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
    return objectSummaries.toArray(new S3ObjectSummary[objectSummaries.size()]);
}

Example 32 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project dataverse by IQSS.

the class S3AccessIO method listAuxObjects.

@Override
public List<String> listAuxObjects() throws IOException {
    if (!this.canWrite()) {
        open();
    }
    String prefix = getDestinationKey("");
    List<String> ret = new ArrayList<>();
    ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    ObjectListing storedAuxFilesList = s3.listObjects(req);
    List<S3ObjectSummary> storedAuxFilesSummary = storedAuxFilesList.getObjectSummaries();
    try {
        while (storedAuxFilesList.isTruncated()) {
            logger.fine("S3 listAuxObjects: going to next page of list");
            storedAuxFilesList = s3.listNextBatchOfObjects(storedAuxFilesList);
            storedAuxFilesSummary.addAll(storedAuxFilesList.getObjectSummaries());
        }
    } catch (AmazonClientException ase) {
        logger.warning("Caught an AmazonServiceException in S3AccessIO.listAuxObjects():    " + ase.getMessage());
        throw new IOException("S3AccessIO: Failed to get aux objects for listing.");
    }
    for (S3ObjectSummary item : storedAuxFilesSummary) {
        String destinationKey = item.getKey();
        String fileName = destinationKey.substring(destinationKey.lastIndexOf(".") + 1);
        logger.fine("S3 cached aux object fileName: " + fileName);
        ret.add(fileName);
    }
    return ret;
}

Example 33 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project stocator by CODAIT.

the class COSAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException {
    FileStatus res = null;
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    String key = pathToKey(path);
    LOG.debug("getFileStatus: on original key {}", key);
    FileStatus fileStatus = null;
    try {
        fileStatus = getFileStatusKeyBased(key, path);
    } catch (AmazonS3Exception e) {
        LOG.warn("file status {} returned {}", key, e.getStatusCode());
        if (e.getStatusCode() != 404) {
            LOG.warn("Throw IOException for {}. Most likely authentication failed", key);
            throw new IOException(e);
        }
    }
    if (fileStatus != null) {
        LOG.trace("getFileStatus(completed) {}", path);
        memoryCache.putFileStatus(path.toString(), fileStatus);
        return fileStatus;
    }
    // probably not needed this call
    if (!key.endsWith("/")) {
        String newKey = key + "/";
        try {
            LOG.debug("getFileStatus: original key not found. Alternative key {}", newKey);
            fileStatus = getFileStatusKeyBased(newKey, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        } else {
            // if here: both key and key/ returned not found.
            // trying to see if pseudo directory of the form
            // a/b/key/d/e (a/b/key/ doesn't exists by itself)
            // perform listing on the key
            LOG.debug("getFileStatus: Modifined key {} not found. Trying to list", key);
            key = maybeAddTrailingSlash(key);
            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(mBucket);
            request.setPrefix(key);
            request.setDelimiter("/");
            request.setMaxKeys(1);
            ObjectListing objects = mClient.listObjects(request);
            if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            } else if (key.isEmpty()) {
                LOG.trace("Found root directory");
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            }
        }
    }
    LOG.debug("Not found {}. Throw FNF exception", path.toString());
    throw new FileNotFoundException("Not found " + path.toString());
}

Example 34 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project stocator by CODAIT.

the class COSAPIClient method list.

@Override
public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased, Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException {
    LOG.debug("Native direct list status for {}", path);
    ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
    String key = pathToKey(path);
    if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/") && !path.toString().equals(hostName)) {
        key = key + "/";
        LOG.debug("listNativeDirect modify key to {}", key);
    }
    Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>();
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(mBucket);
    request.setMaxKeys(5000);
    request.setPrefix(key);
    if (!flatListing) {
        request.setDelimiter("/");
    }
    ObjectListing objectList = mClient.listObjects(request);
    List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries();
    List<String> commonPrefixes = objectList.getCommonPrefixes();
    boolean objectScanContinue = true;
    S3ObjectSummary prevObj = null;
    // start FTA logic
    boolean stocatorOrigin = isSparkOrigin(key, path.toString());
    if (stocatorOrigin) {
        LOG.debug("Stocator origin is true for {}", key);
        if (!isJobSuccessful(key)) {
            LOG.debug("{} created by failed Spark job. Skipped", key);
            if (fModeAutomaticDelete) {
                delete(hostName, new Path(key), true);
            }
            return new FileStatus[0];
        }
    }
    while (objectScanContinue) {
        for (S3ObjectSummary obj : objectSummaries) {
            if (prevObj == null) {
                prevObj = obj;
                prevObj.setKey(correctPlusSign(key, prevObj.getKey()));
                continue;
            }
            obj.setKey(correctPlusSign(key, obj.getKey()));
            String objKey = obj.getKey();
            String unifiedObjectName = extractUnifiedObjectName(objKey);
            LOG.trace("list candidate {}, unified name {}", objKey, unifiedObjectName);
            if (stocatorOrigin && !fullListing) {
                LOG.trace("{} created by Spark", unifiedObjectName);
                // we need to make sure there are no failed attempts
                if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) {
                    // found failed that was not aborted.
                    LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey);
                    if (prevObj.getSize() < obj.getSize()) {
                        LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey());
                        prevObj = obj;
                    }
                    continue;
                }
            }
            FileStatus fs = createFileStatus(prevObj, hostName, path);
            if (fs.getLen() > 0 || fullListing) {
                LOG.trace("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
                if (filter == null) {
                    tmpResult.add(fs);
                } else if (filter != null && filter.accept(fs.getPath())) {
                    tmpResult.add(fs);
                } else {
                    LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
                }
            } else {
                emptyObjects.put(fs.getPath().toString(), fs);
            }
            prevObj = obj;
        }
        boolean isTruncated = objectList.isTruncated();
        if (isTruncated) {
            objectList = mClient.listNextBatchOfObjects(objectList);
            objectSummaries = objectList.getObjectSummaries();
        } else {
            objectScanContinue = false;
        }
    }
    if (prevObj != null) {
        FileStatus fs = createFileStatus(prevObj, hostName, path);
        LOG.trace("Adding the last object from the list {}", fs.getPath());
        if (fs.getLen() > 0 || fullListing) {
            LOG.trace("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
            if (filter == null) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else if (filter != null && filter.accept(fs.getPath())) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else {
                LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
            }
        } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) {
            emptyObjects.put(fs.getPath().toString(), fs);
        }
    }
    // get common prefixes
    for (String comPrefix : commonPrefixes) {
        LOG.trace("Common prefix is {}", comPrefix);
        if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString()) || emptyObjects.isEmpty()) {
            FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix));
            LOG.trace("Match between common prefix and empty object {}. Adding to result", comPrefix);
            if (filter == null) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else if (filter != null && filter.accept(status.getPath())) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else {
                LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(), filter);
            }
        }
    }
    return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}

Example 35 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project testcontainers-java by testcontainers.

the class SimpleLocalstackS3Test method s3Test.

@Test
public void s3Test() throws IOException {
    AmazonS3 s3 = AmazonS3ClientBuilder.standard().withEndpointConfiguration(localstack.getEndpointConfiguration(S3)).withCredentials(localstack.getDefaultCredentialsProvider()).build();
    s3.createBucket("foo");
    s3.putObject("foo", "bar", "baz");
    final List<Bucket> buckets = s3.listBuckets();
    assertEquals("The created bucket is present", 1, buckets.size());
    final Bucket bucket = buckets.get(0);
    assertEquals("The created bucket has the right name", "foo", bucket.getName());
    assertEquals("The created bucket has the right name", "foo", bucket.getName());
    final ObjectListing objectListing = s3.listObjects("foo");
    assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size());
    final S3Object object = s3.getObject("foo", "bar");
    final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8"));
    assertEquals("The object can be retrieved", "baz", content);
}