Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 86 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project syndesis-qe by syndesisio.

the class S3Utils method deleteS3Bucket.

public void deleteS3Bucket(String bucketName) {
    try {
        final ObjectListing bucketObjects = s3client.listObjects(bucketName);
        for (Iterator<?> iterator = bucketObjects.getObjectSummaries().iterator(); iterator.hasNext(); ) {
            final S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
            s3client.deleteObject(bucketName, summary.getKey());
        }
        s3client.deleteBucket(bucketName);
    } catch (AmazonServiceException e) {
        log.error("Could not delete the S3 bucket: {}", e.getErrorMessage());
    }
}

Example 87 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project stocator by SparkTC.

the class COSAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException {
    FileStatus res = null;
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    String key = pathToKey(path);
    LOG.debug("getFileStatus: on original key {}", key);
    FileStatus fileStatus = null;
    try {
        fileStatus = getFileStatusKeyBased(key, path);
    } catch (AmazonS3Exception e) {
        LOG.warn("file status {} returned {}", key, e.getStatusCode());
        if (e.getStatusCode() != 404) {
            LOG.warn("Throw IOException for {}. Most likely authentication failed", key);
            throw new IOException(e);
        }
    }
    if (fileStatus != null) {
        LOG.trace("getFileStatus(completed) {}", path);
        memoryCache.putFileStatus(path.toString(), fileStatus);
        return fileStatus;
    }
    // probably not needed this call
    if (!key.endsWith("/")) {
        String newKey = key + "/";
        try {
            LOG.debug("getFileStatus: original key not found. Alternative key {}", key);
            fileStatus = getFileStatusKeyBased(newKey, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        } else {
            // if here: both key and key/ returned not found.
            // trying to see if pseudo directory of the form
            // a/b/key/d/e (a/b/key/ doesn't exists by itself)
            // perform listing on the key
            LOG.debug("getFileStatus: Modifined key {} not found. Trying to list", key);
            key = maybeAddTrailingSlash(key);
            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(mBucket);
            request.setPrefix(key);
            request.setDelimiter("/");
            request.setMaxKeys(1);
            ObjectListing objects = mClient.listObjects(request);
            if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            } else if (key.isEmpty()) {
                LOG.trace("Found root directory");
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            }
        }
    }
    LOG.debug("Not found {}. Throw FNF exception", path.toString());
    throw new FileNotFoundException("Not found " + path.toString());
}

Example 88 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project apex-malhar by apache.

the class S3RecordReaderModuleAppTest method deleteBucketAndContent.

public void deleteBucketAndContent() {
    // Get the list of objects
    ObjectListing objectListing = client.listObjects(testMeta.bucketKey);
    for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
        S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
        LOG.info("Deleting an object: {}", objectSummary.getKey());
        client.deleteObject(testMeta.bucketKey, objectSummary.getKey());
    }
    client.deleteBucket(testMeta.bucketKey);
}

Example 89 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project dataverse by IQSS.

the class S3AccessIO method deleteAllAuxObjects.

@Override
public void deleteAllAuxObjects() throws IOException {
    if (!this.canWrite()) {
        open(DataAccessOption.WRITE_ACCESS);
    }
    String prefix = getDestinationKey("");
    List<S3ObjectSummary> storedAuxFilesSummary = null;
    try {
        ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
        ObjectListing storedAuxFilesList = s3.listObjects(req);
        storedAuxFilesSummary = storedAuxFilesList.getObjectSummaries();
        while (storedAuxFilesList.isTruncated()) {
            storedAuxFilesList = s3.listNextBatchOfObjects(storedAuxFilesList);
            storedAuxFilesSummary.addAll(storedAuxFilesList.getObjectSummaries());
        }
    } catch (AmazonClientException ase) {
        logger.warning("Caught an AmazonServiceException:    " + ase.getMessage());
        throw new IOException("S3AccessIO: Failed to get aux objects for listing to delete.");
    }
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName);
    List<KeyVersion> keys = new ArrayList<>();
    for (S3ObjectSummary item : storedAuxFilesSummary) {
        String destinationKey = item.getKey();
        keys.add(new KeyVersion(destinationKey));
    }
    // Check if the list of auxiliary files for a data file is empty
    if (keys.isEmpty()) {
        logger.fine("S3AccessIO: No auxiliary objects to delete.");
        return;
    }
    multiObjectDeleteRequest.setKeys(keys);
    logger.fine("Trying to delete auxiliary files...");
    try {
        s3.deleteObjects(multiObjectDeleteRequest);
    } catch (MultiObjectDeleteException e) {
        logger.warning("S3AccessIO: Unable to delete auxilary objects" + e.getMessage());
        throw new IOException("S3AccessIO: Failed to delete one or more auxiliary objects.");
    }
}

Example 90 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project molgenis by molgenis.

the class AmazonBucketClientImpl method getMostRecentMatchingKey.

// in case of an key expression all matching keys are collected and the most recent file is downloaded.
private String getMostRecentMatchingKey(AmazonS3 s3Client, String bucketName, String regex) {
    ObjectListing objectListing = s3Client.listObjects(bucketName);
    TreeMap<Date, String> keys = new TreeMap<>();
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().matches(regex)) {
            keys.put(objectSummary.getLastModified(), objectSummary.getKey());
        }
    }
    if (keys.size() == 0)
        throw new MolgenisDataException("No key matching regular expression: " + regex);
    return keys.lastEntry().getValue();
}