Examples with S3ObjectSummary com.amazonaws.services.s3.model.S3ObjectSummary used on opensource projects

Example 46 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project jackrabbit by apache.

the class S3Backend method getAllIdentifiers.

@Override
public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
    long start = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        Set<DataIdentifier> ids = new HashSet<DataIdentifier>();
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                String id = getIdentifierName(s3ObjSumm.getKey());
                if (id != null) {
                    ids.add(new DataIdentifier(id));
                }
            }
            if (!prevObjectListing.isTruncated())
                break;
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        LOG.debug("getAllIdentifiers returned size [{}] took [{}] ms.", ids.size(), (System.currentTimeMillis() - start));
        return ids.iterator();
    } catch (AmazonServiceException e) {
        throw new DataStoreException("Could not list objects", e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 47 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project jackrabbit by apache.

the class S3Backend method deleteAllOlderThan.

@Override
public Set<DataIdentifier> deleteAllOlderThan(long min) throws DataStoreException {
    long start = System.currentTimeMillis();
    // S3 stores lastModified to lower boundary of timestamp in ms.
    // and hence min is reduced by 1000ms.
    min = min - 1000;
    Set<DataIdentifier> deleteIdSet = new HashSet<DataIdentifier>(30);
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        while (true) {
            List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                DataIdentifier identifier = new DataIdentifier(getIdentifierName(s3ObjSumm.getKey()));
                long lastModified = s3ObjSumm.getLastModified().getTime();
                LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
                if (lastModified < min && getDataStore().confirmDelete(identifier) && //  order is important here
                s3service.getObjectMetadata(bucket, s3ObjSumm.getKey()).getLastModified().getTime() < min) {
                    getDataStore().deleteFromCache(identifier);
                    LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                    deleteIdSet.add(identifier);
                }
            }
            if (deleteList.size() > 0) {
                DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                delObjsReq.setKeys(deleteList);
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                if (dobjs.getDeletedObjects().size() != deleteList.size()) {
                    throw new DataStoreException("Incomplete delete object request. only  " + dobjs.getDeletedObjects().size() + " out of " + deleteList.size() + " are deleted");
                } else {
                    LOG.debug("[{}] records deleted from datastore", deleteList);
                }
            }
            if (!prevObjectListing.isTruncated()) {
                break;
            }
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.info("deleteAllOlderThan: min=[{}] exit. Deleted[{}] records. Number of records deleted [{}] took [{}]ms", new Object[] { min, deleteIdSet, deleteIdSet.size(), (System.currentTimeMillis() - start) });
    return deleteIdSet;
}

Example 48 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project jackrabbit-oak by apache.

the class S3DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = getS3Config();
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 49 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project jackrabbit-oak by apache.

the class S3Backend method getAllMetadataRecords.

public List<DataRecord> getAllMetadataRecords(String prefix) {
    List<DataRecord> metadataList = new ArrayList<DataRecord>();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
        for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            metadataList.add(new S3DataRecord(s3service, bucket, stripMetaKeyPrefix(s3ObjSumm.getKey()), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true));
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return metadataList;
}

Example 50 with S3ObjectSummary

use of com.amazonaws.services.s3.model.S3ObjectSummary in project jackrabbit-oak by apache.

the class S3Backend method getAllMetadataRecords.

@Override
public List<DataRecord> getAllMetadataRecords(String prefix) {
    List<DataRecord> metadataList = new ArrayList<DataRecord>();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
        for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            metadataList.add(new S3DataRecord(this, s3service, bucket, new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true));
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return metadataList;
}