use of org.apache.jackrabbit.core.data.util.NamedThreadFactory in project jackrabbit by apache.
the class CachingDataStore method init.
/**
* Initialized the data store. If the path is not set, <repository
* home>/repository/datastore is used. This directory is automatically
* created if it does not yet exist. During first initialization, it upload
* all files from local datastore to backed and local datastore act as a
* local cache.
*/
@Override
public void init(String homeDir) throws RepositoryException {
try {
if (path == null) {
path = homeDir + "/repository/datastore";
}
// create tmp inside path
tmpDir = new File(path, "tmp");
LOG.info("path=[{}], tmpPath=[{}]", path, tmpDir.getAbsolutePath());
directory = new File(path);
mkdirs(directory);
mkdirs(new File(homeDir));
if (!mkdirs(tmpDir)) {
FileUtils.cleanDirectory(tmpDir);
LOG.info("tmp=[{}] cleaned.", tmpDir.getPath());
}
boolean asyncWriteCacheInitStatus = true;
try {
asyncWriteCache = new AsyncUploadCache();
asyncWriteCache.init(homeDir, path, asyncUploadLimit);
} catch (Exception e) {
LOG.warn("Failed to initialize asyncWriteCache", e);
asyncWriteCacheInitStatus = false;
}
backend = createBackend();
backend.init(this, path, config);
String markerFileName = getMarkerFile();
if (markerFileName != null && !"".equals(markerFileName.trim())) {
// create marker file in homeDir to avoid deletion in cache
// cleanup.
File markerFile = new File(homeDir, markerFileName);
if (!markerFile.exists()) {
LOG.info("load files from local cache");
uploadFilesFromCache();
try {
markerFile.createNewFile();
} catch (IOException e) {
throw new DataStoreException("Could not create marker file " + markerFile.getAbsolutePath(), e);
}
} else {
LOG.info("marker file = [{}] exists ", markerFile.getAbsolutePath());
if (!asyncWriteCacheInitStatus) {
LOG.info("Initialization of asyncWriteCache failed. " + "Re-loading all files from local cache");
uploadFilesFromCache();
asyncWriteCache.reset();
}
}
} else {
throw new DataStoreException("Failed to intialized DataStore." + " MarkerFileName is null or empty. ");
}
// upload any leftover async uploads to backend during last shutdown
Set<String> fileList = asyncWriteCache.getAll();
if (fileList != null && !fileList.isEmpty()) {
List<String> errorFiles = new ArrayList<String>();
LOG.info("Uploading [{}] and size=[{}] from AsyncUploadCache.", fileList, fileList.size());
long totalSize = 0;
List<File> files = new ArrayList<File>(fileList.size());
for (String fileName : fileList) {
File f = new File(path, fileName);
if (!f.exists()) {
errorFiles.add(fileName);
LOG.error("Cannot upload pending file [{}]. File doesn't exist.", f.getAbsolutePath());
} else {
totalSize += f.length();
files.add(new File(path, fileName));
}
}
new FilesUploader(files, totalSize, concurrentUploadsThreads, true).upload();
if (!continueOnAsyncUploadFailure && errorFiles.size() > 0) {
LOG.error("Pending uploads of files [{}] failed. Files do not exist in Local cache.", errorFiles);
LOG.error("To continue set [continueOnAsyncUploadFailure] " + "to true in Datastore configuration in " + "repository.xml. There would be inconsistent data " + "in repository due the missing files. ");
throw new RepositoryException("Cannot upload async uploads from local cache. Files not found.");
} else {
if (errorFiles.size() > 0) {
LOG.error("Pending uploads of files [{}] failed. Files do" + " not exist in Local cache. Continuing as " + "[continueOnAsyncUploadFailure] is set to true.", errorFiles);
}
LOG.info("Reseting AsyncWrite Cache list.");
asyncWriteCache.reset();
}
}
downloadExecService = Executors.newFixedThreadPool(5, new NamedThreadFactory("backend-file-download-worker"));
cache = new LocalCache(path, tmpDir.getAbsolutePath(), cacheSize, cachePurgeTrigFactor, cachePurgeResizeFactor, asyncWriteCache);
/*
* Initialize LRU cache of size {@link #recLengthCacheSize}
*/
recLenCache = Collections.synchronizedMap(new LinkedHashMap<DataIdentifier, Long>(recLengthCacheSize, 0.75f, true) {
private static final long serialVersionUID = -8752749075395630485L;
@Override
protected boolean removeEldestEntry(Map.Entry<DataIdentifier, Long> eldest) {
if (size() > recLengthCacheSize) {
LOG.trace("evicted from recLengthCache [{}]", eldest.getKey());
return true;
}
return false;
}
});
} catch (Exception e) {
throw new RepositoryException(e);
}
}
use of org.apache.jackrabbit.core.data.util.NamedThreadFactory in project jackrabbit by apache.
the class S3Backend method renameKeys.
/**
* This method rename object keys in S3 concurrently. The number of
* concurrent threads is defined by 'maxConnections' property in
* aws.properties. As S3 doesn't have "move" command, this method simulate
* move as copy object object to new key and then delete older key.
*/
private void renameKeys() throws DataStoreException {
long startTime = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
long count = 0;
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectListing prevObjectListing = s3service.listObjects(bucket);
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
boolean taskAdded = false;
while (true) {
for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
taskAdded = true;
count++;
// delete the object if it follows old key name format
if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
}
}
if (!prevObjectListing.isTruncated())
break;
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
// This will make the executor accept no new threads
// and finish all existing threads in the queue
executor.shutdown();
try {
// Wait until all threads are finish
while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
LOG.info("Rename S3 keys tasks timedout. Waiting again");
}
} catch (InterruptedException ie) {
}
LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
// Delete older keys.
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
int batchSize = 500, startIndex = 0, size = deleteList.size();
int endIndex = batchSize < size ? batchSize : size;
while (endIndex <= size) {
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", new Object[] { dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1) });
if (endIndex == size) {
break;
} else {
startIndex = endIndex;
endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
}
}
}
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of org.apache.jackrabbit.core.data.util.NamedThreadFactory in project jackrabbit-oak by apache.
the class S3Backend method init.
public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
startTime = new Date();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("init");
this.store = store;
s3ReqDecorator = new S3RequestDecorator(prop);
s3service = Utils.openService(prop);
if (bucket == null || "".equals(bucket.trim())) {
bucket = prop.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (Strings.isNullOrEmpty(bucket)) {
bucket = prop.getProperty(S3Constants.S3_CONTAINER);
}
}
String region = prop.getProperty(S3Constants.S3_REGION);
Region s3Region = null;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
if (ec2Region != null) {
s3Region = Region.fromValue(ec2Region.getName());
} else {
throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
}
} else {
if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
s3Region = Region.US_Standard;
} else if (Region.EU_Ireland.toString().equals(region)) {
s3Region = Region.EU_Ireland;
} else {
s3Region = Region.fromValue(region);
}
}
if (!s3service.doesBucketExist(bucket)) {
s3service.createBucket(bucket, s3Region);
LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
int writeThreads = 10;
String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
if (writeThreadsStr != null) {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
int asyncWritePoolSize = 10;
String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
if (maxConnsStr != null) {
asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
}
asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize, new NamedThreadFactory("s3-write-worker"));
String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
}
LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
LOG.debug(" error ", e);
Map<String, String> filteredMap = Maps.newHashMap();
if (prop != null) {
filteredMap = Maps.filterKeys(Maps.fromProperties(prop), new Predicate<String>() {
@Override
public boolean apply(String input) {
return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
}
});
}
throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of org.apache.jackrabbit.core.data.util.NamedThreadFactory in project jackrabbit-oak by apache.
the class S3Backend method init.
public void init() throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
startTime = new Date();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("init");
s3ReqDecorator = new S3RequestDecorator(properties);
s3service = Utils.openService(properties);
if (bucket == null || "".equals(bucket.trim())) {
bucket = properties.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (Strings.isNullOrEmpty(bucket)) {
bucket = properties.getProperty(S3Constants.S3_CONTAINER);
}
}
secret = properties.getProperty("secret");
String region = properties.getProperty(S3Constants.S3_REGION);
Region s3Region;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
if (ec2Region != null) {
s3Region = Region.fromValue(ec2Region.getName());
} else {
throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
}
} else {
if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
s3Region = Region.US_Standard;
} else if (Region.EU_Ireland.toString().equals(region)) {
s3Region = Region.EU_Ireland;
} else {
s3Region = Region.fromValue(region);
}
}
if (!s3service.doesBucketExist(bucket)) {
s3service.createBucket(bucket, s3Region);
LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
int writeThreads = 10;
String writeThreadsStr = properties.getProperty(S3Constants.S3_WRITE_THREADS);
if (writeThreadsStr != null) {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
tmx = new TransferManager(s3service, Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
String renameKeyProp = properties.getProperty(S3Constants.S3_RENAME_KEYS);
boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
}
LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
LOG.debug(" error ", e);
Map<String, String> filteredMap = Maps.newHashMap();
if (properties != null) {
filteredMap = Maps.filterKeys(Maps.fromProperties(properties), new Predicate<String>() {
@Override
public boolean apply(String input) {
return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
}
});
}
throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of org.apache.jackrabbit.core.data.util.NamedThreadFactory in project jackrabbit-oak by apache.
the class S3Backend method renameKeys.
/**
* This method rename object keys in S3 concurrently. The number of
* concurrent threads is defined by 'maxConnections' property in
* aws.properties. As S3 doesn't have "move" command, this method simulate
* move as copy object object to new key and then delete older key.
*/
private void renameKeys() throws DataStoreException {
long startTime = System.currentTimeMillis();
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
long count = 0;
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectListing prevObjectListing = s3service.listObjects(bucket);
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
boolean taskAdded = false;
while (true) {
for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
taskAdded = true;
count++;
// delete the object if it follows old key name format
if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
}
}
if (!prevObjectListing.isTruncated())
break;
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
// This will make the executor accept no new threads
// and finish all existing threads in the queue
executor.shutdown();
try {
// Wait until all threads are finish
while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
LOG.info("Rename S3 keys tasks timedout. Waiting again");
}
} catch (InterruptedException ie) {
}
LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
// Delete older keys.
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
int batchSize = 500, startIndex = 0, size = deleteList.size();
int endIndex = batchSize < size ? batchSize : size;
while (endIndex <= size) {
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1));
if (endIndex == size) {
break;
} else {
startIndex = endIndex;
endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
}
}
}
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
Aggregations