use of com.amazonaws.services.s3.model.Bucket in project jackrabbit-oak by apache.
the class S3Backend method getLastModified.
@Override
public long getLastModified(DataIdentifier identifier) throws DataStoreException {
long start = System.currentTimeMillis();
String key = getKeyName(identifier);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectMetadata object = s3service.getObjectMetadata(bucket, key);
long lastModified = object.getLastModified().getTime();
LOG.debug("Identifier [{}]'s lastModified = [{}] took [{}]ms.", new Object[] { identifier, lastModified, (System.currentTimeMillis() - start) });
return lastModified;
} catch (AmazonServiceException e) {
if (e.getStatusCode() == 404 || e.getStatusCode() == 403) {
LOG.info("getLastModified:Identifier [{}] not found. Took [{}] ms.", identifier, (System.currentTimeMillis() - start));
}
throw new DataStoreException(e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of com.amazonaws.services.s3.model.Bucket in project jackrabbit-oak by apache.
the class S3Backend method exists.
@Override
public boolean exists(DataIdentifier identifier, boolean touch) throws DataStoreException {
long start = System.currentTimeMillis();
String key = getKeyName(identifier);
ObjectMetadata objectMetaData = null;
boolean retVal = false;
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
objectMetaData = s3service.getObjectMetadata(bucket, key);
if (objectMetaData != null) {
retVal = true;
if (touch) {
CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key);
copReq.setNewObjectMetadata(objectMetaData);
Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq));
copy.waitForCopyResult();
LOG.debug("[{}] touched took [{}] ms. ", identifier, (System.currentTimeMillis() - start));
}
} else {
retVal = false;
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() == 404 || e.getStatusCode() == 403) {
retVal = false;
} else {
throw new DataStoreException("Error occured to find exists for key [" + identifier.toString() + "]", e);
}
} catch (Exception e) {
throw new DataStoreException("Error occured to find exists for key " + identifier.toString(), e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
LOG.debug("exists [{}]: [{}] took [{}] ms.", new Object[] { identifier, retVal, (System.currentTimeMillis() - start) });
return retVal;
}
use of com.amazonaws.services.s3.model.Bucket in project jackrabbit-oak by apache.
the class S3Backend method deleteAllOlderThan.
@Override
public Set<DataIdentifier> deleteAllOlderThan(long min) throws DataStoreException {
long start = System.currentTimeMillis();
// S3 stores lastModified to lower boundary of timestamp in ms.
// and hence min is reduced by 1000ms.
min = min - 1000;
Set<DataIdentifier> deleteIdSet = new HashSet<DataIdentifier>(30);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectListing prevObjectListing = s3service.listObjects(bucket);
while (true) {
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
if (!s3ObjSumm.getKey().startsWith(META_KEY_PREFIX)) {
DataIdentifier identifier = new DataIdentifier(getIdentifierName(s3ObjSumm.getKey()));
long lastModified = s3ObjSumm.getLastModified().getTime();
LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
if (lastModified < min && store.confirmDelete(identifier) && // order is important here
s3service.getObjectMetadata(bucket, s3ObjSumm.getKey()).getLastModified().getTime() < min) {
store.deleteFromCache(identifier);
LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
deleteIdSet.add(identifier);
}
}
}
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
delObjsReq.setKeys(deleteList);
DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
if (dobjs.getDeletedObjects().size() != deleteList.size()) {
throw new DataStoreException("Incomplete delete object request. only " + dobjs.getDeletedObjects().size() + " out of " + deleteList.size() + " are deleted");
} else {
LOG.debug("[{}] records deleted from datastore", deleteList);
}
}
if (!prevObjectListing.isTruncated()) {
break;
}
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
LOG.info("deleteAllOlderThan: min=[{}] exit. Deleted[{}] records. Number of records deleted [{}] took [{}]ms", new Object[] { min, deleteIdSet, deleteIdSet.size(), (System.currentTimeMillis() - start) });
return deleteIdSet;
}
use of com.amazonaws.services.s3.model.Bucket in project jackrabbit-oak by apache.
the class S3Backend method getRecord.
@Override
public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
long start = System.currentTimeMillis();
String key = getKeyName(identifier);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectMetadata object = s3service.getObjectMetadata(bucket, key);
S3DataRecord record = new S3DataRecord(s3service, bucket, identifier.toString(), object.getLastModified().getTime(), object.getContentLength());
LOG.debug("Identifier [{}]'s getRecord = [{}] took [{}]ms.", new Object[] { identifier, record, (System.currentTimeMillis() - start) });
return record;
} catch (AmazonServiceException e) {
if (e.getStatusCode() == 404 || e.getStatusCode() == 403) {
LOG.info("getRecord:Identifier [{}] not found. Took [{}] ms.", identifier, (System.currentTimeMillis() - start));
}
throw new DataStoreException(e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of com.amazonaws.services.s3.model.Bucket in project jackrabbit-oak by apache.
the class S3Backend method init.
public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
startTime = new Date();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("init");
this.store = store;
s3ReqDecorator = new S3RequestDecorator(prop);
s3service = Utils.openService(prop);
if (bucket == null || "".equals(bucket.trim())) {
bucket = prop.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (Strings.isNullOrEmpty(bucket)) {
bucket = prop.getProperty(S3Constants.S3_CONTAINER);
}
}
String region = prop.getProperty(S3Constants.S3_REGION);
Region s3Region = null;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
if (ec2Region != null) {
s3Region = Region.fromValue(ec2Region.getName());
} else {
throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
}
} else {
if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
s3Region = Region.US_Standard;
} else if (Region.EU_Ireland.toString().equals(region)) {
s3Region = Region.EU_Ireland;
} else {
s3Region = Region.fromValue(region);
}
}
if (!s3service.doesBucketExist(bucket)) {
s3service.createBucket(bucket, s3Region);
LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
int writeThreads = 10;
String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
if (writeThreadsStr != null) {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
int asyncWritePoolSize = 10;
String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
if (maxConnsStr != null) {
asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
}
asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize, new NamedThreadFactory("s3-write-worker"));
String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
}
LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
LOG.debug(" error ", e);
Map<String, String> filteredMap = Maps.newHashMap();
if (prop != null) {
filteredMap = Maps.filterKeys(Maps.fromProperties(prop), new Predicate<String>() {
@Override
public boolean apply(String input) {
return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
}
});
}
throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
Aggregations