use of com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException in project ksan by infinistor.
the class MongoDataRepository method getBucketList.
@Override
public List<Bucket> getBucketList() {
List<Bucket> btList = new ArrayList();
FindIterable<Document> fit = buckets.find();
Iterator it = fit.iterator();
while ((it.hasNext())) {
try {
Document doc = (Document) it.next();
String bucketName = (String) doc.get(BUCKETNAME);
// String diskPoolId = (String)doc.get("diskPoolId");
// String bucketId = (String)doc.get("bucketId");
// new Bucket(bucketName, bucketId, diskPoolId);
Bucket bt = parseBucket(bucketName, doc);
btList.add(bt);
} catch (ResourceNotFoundException | SQLException ex) {
Logger.getLogger(MongoDataRepository.class.getName()).log(Level.SEVERE, null, ex);
}
}
return btList;
}
use of com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException in project ksan by infinistor.
the class MongoDataRepository method getObjectList.
@Override
public List<Metadata> getObjectList(String bucketName, Object query, int maxKeys) throws SQLException {
String diskPoolId = "1";
MongoCollection<Document> objects;
objects = database.getCollection(bucketName);
BasicDBObject sortBy = new BasicDBObject(OBJKEY, 1);
BasicDBObject mongoQuery = (BasicDBObject) query;
if (!mongoQuery.containsField(LASTVERSION)) {
sortBy.append(LASTMODIFIED, -1);
sortBy.append("_id", -1);
}
FindIterable<Document> oit = objects.find(mongoQuery).limit(maxKeys).sort(sortBy);
Iterator it = oit.iterator();
List<Metadata> list = new ArrayList();
Bucket bt = obmCache.getBucketFromCache(bucketName);
if (bt != null)
diskPoolId = bt.getDiskPoolId();
while ((it.hasNext())) {
Document doc = (Document) it.next();
String key = doc.getString(OBJKEY);
String etag = doc.getString(ETAG);
String tag = doc.getString(TAG);
String meta = doc.getString(META);
String acl = doc.getString(ACL);
String versionid = doc.getString(VERSIONID);
String deletem = doc.getString(DELETEMARKER);
boolean lastversion = doc.getBoolean(LASTVERSION);
String pdiskid = doc.getString(PDISKID);
String rdiskid = doc.getString(RDISKID);
long lastModified = doc.getLong(LASTMODIFIED);
long size = doc.getLong(SIZE);
Metadata mt = new Metadata(bucketName, key);
mt.setVersionId(versionid, deletem, lastversion);
mt.set(etag, tag, meta, acl, size);
mt.setLastModified(lastModified);
try {
mt.setPrimaryDisk(obmCache.getDiskWithId(diskPoolId, pdiskid));
} catch (ResourceNotFoundException ex) {
mt.setPrimaryDisk(new DISK());
}
try {
if (rdiskid == null)
mt.setReplicaDISK(new DISK());
else if (rdiskid.isEmpty())
mt.setReplicaDISK(new DISK());
else
mt.setReplicaDISK(obmCache.getDiskWithId(diskPoolId, rdiskid));
} catch (ResourceNotFoundException ex) {
mt.setReplicaDISK(new DISK());
}
list.add(mt);
}
return list;
}
use of com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException in project ksan by infinistor.
the class S3ObjectOperation method putObjectNormal.
private S3Object putObjectNormal(long length, InputStream is) throws GWException {
S3Object s3Object = new S3Object();
File filePrimary = null;
File tmpFilePrimary = null;
FileOutputStream fosPrimary = null;
File fileReplica = null;
File tmpFileReplica = null;
FileOutputStream fosReplica = null;
File trashPrimary = null;
File trashReplica = null;
OSDClient clientPrimary = null;
OSDClient clientReplica = null;
long totalReads = 0L;
long existFileSize = 0L;
long putSize = 0L;
long calSize = 0L;
try {
MessageDigest md5er = MessageDigest.getInstance(GWConstants.MD5);
byte[] buffer = new byte[GWConstants.MAXBUFSIZE];
int readLength = 0;
existFileSize = objMeta.getSize();
putSize = length;
boolean isPrimaryCache = false;
boolean isReplicaCache = false;
logger.debug("performance mode : {}", GWConfig.getPerformanceMode());
logger.debug("objMeta - replicaCount : {}", objMeta.getReplicaCount());
// No option
if (GWConfig.isNoOption()) {
if (objMeta.getReplicaCount() > 1) {
existFileSize *= objMeta.getReplicaCount();
putSize *= objMeta.getReplicaCount();
logger.debug("bucket : {}, object : {}", objMeta.getBucket(), objMeta.getPath());
logger.debug("primary disk id : {}, osd ip : {}", objMeta.getPrimaryDisk().getId(), objMeta.getPrimaryDisk().getOsdIp());
logger.debug("replica disk id : {}, osd ip : {}", objMeta.getReplicaDisk().getId(), objMeta.getReplicaDisk().getOsdIp());
if (GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
if (!Strings.isNullOrEmpty(GWConfig.getCacheDisk()) && length <= (GWConfig.getCacheFileSize() * GWConstants.MEGABYTES)) {
filePrimary = new File(makeCachePath(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
tmpFilePrimary = new File(makeCachePath(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
trashPrimary = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
File file = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(filePrimary);
com.google.common.io.Files.createParentDirs(tmpFilePrimary);
logger.debug("filePrimary path : {}", filePrimary.getAbsolutePath());
logger.debug("tmpFilePrimary path : {}", tmpFilePrimary.getAbsolutePath());
fosPrimary = new FileOutputStream(tmpFilePrimary, false);
isPrimaryCache = true;
} else {
filePrimary = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
tmpFilePrimary = new File(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
trashPrimary = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(filePrimary);
com.google.common.io.Files.createParentDirs(tmpFilePrimary);
fosPrimary = new FileOutputStream(tmpFilePrimary, false);
}
} else {
clientPrimary = OSDClientManager.getInstance().getOSDClient(objMeta.getPrimaryDisk().getOsdIp());
clientPrimary.putInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId(), GWConfig.getPerformanceMode());
}
if (GWUtils.getLocalIP().equals(objMeta.getReplicaDisk().getOsdIp())) {
if (!Strings.isNullOrEmpty(GWConfig.getCacheDisk()) && length <= GWConfig.getCacheFileSize() * GWConstants.MEGABYTES) {
fileReplica = new File(makeCachePath(makeObjPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId)));
tmpFileReplica = new File(makeCachePath(makeTempPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId)));
trashReplica = new File(makeCachePath(makeTrashPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId)));
File file = new File(makeObjPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(fileReplica);
com.google.common.io.Files.createParentDirs(tmpFileReplica);
logger.debug("fileReplica path : {}", fileReplica.getAbsolutePath());
logger.debug("tmpFileReplica path : {}", tmpFileReplica.getAbsolutePath());
fosReplica = new FileOutputStream(tmpFileReplica, false);
isReplicaCache = true;
} else {
fileReplica = new File(makeObjPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
tmpFileReplica = new File(makeTempPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
trashReplica = new File(makeTrashPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(fileReplica);
com.google.common.io.Files.createParentDirs(tmpFileReplica);
fosReplica = new FileOutputStream(tmpFileReplica, false);
}
} else {
clientReplica = OSDClientManager.getInstance().getOSDClient(objMeta.getReplicaDisk().getOsdIp());
clientReplica.putInit(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRIBUTE_REPLICATION_REPLICA, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL, GWConfig.getPerformanceMode());
}
while ((readLength = is.read(buffer, 0, GWConstants.BUFSIZE)) >= 0) {
totalReads += readLength;
if (filePrimary == null) {
clientPrimary.put(buffer, 0, readLength);
} else {
fosPrimary.write(buffer, 0, readLength);
}
if (fileReplica == null) {
clientReplica.put(buffer, 0, readLength);
} else {
fosReplica.write(buffer, 0, readLength);
}
md5er.update(buffer, 0, readLength);
}
if (filePrimary == null) {
clientPrimary.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientPrimary);
} else {
fosPrimary.flush();
if (filePrimary.exists()) {
retryRenameTo(filePrimary, trashPrimary);
}
setAttributeFileReplication(tmpFilePrimary, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId());
retryRenameTo(tmpFilePrimary, filePrimary);
if (isPrimaryCache) {
String path = makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId);
Files.createSymbolicLink(Paths.get(path), Paths.get(filePrimary.getAbsolutePath()));
}
}
if (fileReplica == null) {
clientReplica.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientReplica);
} else {
fosReplica.flush();
if (fileReplica.exists()) {
retryRenameTo(fileReplica, trashReplica);
}
setAttributeFileReplication(tmpFileReplica, GWConstants.FILE_ATTRIBUTE_REPLICATION_REPLICA, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL);
retryRenameTo(tmpFileReplica, fileReplica);
if (isReplicaCache) {
String path = makeObjPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId);
Files.createSymbolicLink(Paths.get(path), Paths.get(fileReplica.getAbsolutePath()));
}
}
} else {
File file = null;
File tmpFile = null;
File trashFile = null;
if (GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
if (!Strings.isNullOrEmpty(GWConfig.getCacheDisk()) && length <= GWConfig.getCacheFileSize() * GWConstants.MEGABYTES) {
file = new File(makeCachePath(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
tmpFile = new File(makeCachePath(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
trashFile = new File(makeCachePath(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
File link = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(tmpFile);
com.google.common.io.Files.createParentDirs(link);
fosPrimary = new FileOutputStream(tmpFile, false);
isPrimaryCache = true;
} else {
file = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
tmpFile = new File(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
trashFile = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(tmpFile);
fosPrimary = new FileOutputStream(tmpFile, false);
}
} else {
clientPrimary = OSDClientManager.getInstance().getOSDClient(objMeta.getPrimaryDisk().getOsdIp());
clientPrimary.putInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId(), GWConfig.getPerformanceMode());
}
while ((readLength = is.read(buffer, 0, GWConstants.BUFSIZE)) >= 0) {
totalReads += readLength;
if (file == null) {
clientPrimary.put(buffer, 0, readLength);
} else {
fosPrimary.write(buffer, 0, readLength);
}
md5er.update(buffer, 0, readLength);
}
if (file == null) {
clientPrimary.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientPrimary);
} else {
fosPrimary.flush();
if (file.exists()) {
retryRenameTo(file, trashFile);
}
if (objMeta.getReplicaDisk() != null) {
setAttributeFileReplication(tmpFile, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId());
}
retryRenameTo(tmpFile, file);
if (isPrimaryCache) {
String path = makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId);
Files.createSymbolicLink(Paths.get(path), Paths.get(file.getAbsolutePath()));
}
}
}
} else // No replication option
if (GWConfig.isNoReplica()) {
File file = null;
File tmpFile = null;
File trashFile = null;
if (GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
if (!Strings.isNullOrEmpty(GWConfig.getCacheDisk()) && length <= GWConfig.getCacheFileSize() * GWConstants.MEGABYTES) {
file = new File(makeCachePath(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
tmpFile = new File(makeCachePath(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
trashFile = new File(makeCachePath(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId)));
File link = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(tmpFile);
com.google.common.io.Files.createParentDirs(link);
fosPrimary = new FileOutputStream(tmpFile, false);
isPrimaryCache = true;
} else {
file = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
tmpFile = new File(makeTempPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
trashFile = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
com.google.common.io.Files.createParentDirs(file);
com.google.common.io.Files.createParentDirs(tmpFile);
fosPrimary = new FileOutputStream(tmpFile, false);
}
} else {
clientPrimary = OSDClientManager.getInstance().getOSDClient(objMeta.getPrimaryDisk().getOsdIp());
clientPrimary.putInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId(), GWConfig.getPerformanceMode());
}
while ((readLength = is.read(buffer, 0, GWConstants.BUFSIZE)) >= 0) {
totalReads += readLength;
if (file == null) {
clientPrimary.put(buffer, 0, readLength);
} else {
fosPrimary.write(buffer, 0, readLength);
}
md5er.update(buffer, 0, readLength);
}
if (file == null) {
clientPrimary.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientPrimary);
} else {
fosPrimary.flush();
if (file.exists()) {
retryRenameTo(file, trashFile);
}
if (objMeta.getReplicaDisk() != null) {
setAttributeFileReplication(tmpFile, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId());
}
retryRenameTo(tmpFile, file);
if (isPrimaryCache) {
String path = makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId);
Files.createSymbolicLink(Paths.get(path), Paths.get(file.getAbsolutePath()));
}
}
} else // No IO option
if (GWConfig.isNoIO()) {
while ((readLength = is.read(buffer, 0, GWConstants.BUFSIZE)) > 0) {
totalReads += readLength;
md5er.update(buffer, 0, readLength);
}
} else // No disk option
if (GWConfig.isNoDisk()) {
if (!GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
clientPrimary = OSDClientManager.getInstance().getOSDClient(objMeta.getPrimaryDisk().getOsdIp());
clientPrimary.putInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId(), GWConfig.getPerformanceMode());
}
if (objMeta.getReplicaDisk() != null && !GWUtils.getLocalIP().equals(objMeta.getReplicaDisk().getOsdIp())) {
clientReplica = OSDClientManager.getInstance().getOSDClient(objMeta.getReplicaDisk().getOsdIp());
clientReplica.putInit(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId, length, GWConstants.FILE_ATTRIBUTE_REPLICATION_REPLICA, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL, GWConfig.getPerformanceMode());
}
while ((readLength = is.read(buffer, 0, GWConstants.BUFSIZE)) >= 0) {
totalReads += readLength;
if (clientPrimary != null) {
clientPrimary.put(buffer, 0, readLength);
}
if (clientReplica != null) {
clientReplica.put(buffer, 0, readLength);
}
md5er.update(buffer, 0, readLength);
}
if (clientPrimary != null) {
clientPrimary.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientPrimary);
}
if (clientReplica != null) {
clientReplica.putFlush();
OSDClientManager.getInstance().returnOSDClient(clientReplica);
}
} else {
logger.error(GWConstants.LOG_S3OBJECT_OPERATION_OPTION_NO_CASE, GWConfig.getPerformanceMode());
throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
}
byte[] digest = md5er.digest();
String eTag = base16().lowerCase().encode(digest);
s3Object.setEtag(eTag);
s3Object.setLastModified(new Date());
s3Object.setFileSize(totalReads);
s3Object.setVersionId(versionId);
s3Object.setDeleteMarker(GWConstants.OBJECT_TYPE_FILE);
calSize = putSize - existFileSize;
if (GWConfig.isNoOption()) {
updateBucketUsed(objMeta.getBucket(), calSize);
}
} catch (NoSuchAlgorithmException | IOException e) {
PrintStack.logging(logger, e);
throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
} catch (ResourceNotFoundException e) {
PrintStack.logging(logger, e);
throw new GWException(GWErrorCode.NO_SUCH_KEY, s3Parameter);
} catch (Exception e) {
PrintStack.logging(logger, e);
throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
} finally {
if (objMeta.getReplicaCount() > 1) {
if (fosPrimary != null) {
try {
fosPrimary.close();
} catch (IOException e) {
PrintStack.logging(logger, e);
throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
}
}
if (fosReplica != null) {
try {
fosReplica.close();
} catch (IOException e) {
PrintStack.logging(logger, e);
throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
}
}
}
}
return s3Object;
}
use of com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException in project ksan by infinistor.
the class ObjManager method allocReplicaDisk.
/**
* It will allocate a replica disk for recovery of failed replica object
* @param bucketName bucket name
* @param dpath primary disk path
* @param diskid primary disk disk Id number
* @return new DISK object
* @throws ResourceNotFoundException if there is no server or disk available
* @throws AllServiceOfflineException if all server are offline
* or if all DISK are not Good state
*/
public DISK allocReplicaDisk(String bucketName, String dpath, String diskid) throws ResourceNotFoundException, AllServiceOfflineException {
DISK primary = new DISK();
if (dpath == null && diskid == null)
return null;
if (dpath != null && diskid != null) {
if (dpath.isEmpty() && diskid.isEmpty())
return null;
}
primary.setPath(dpath);
primary.setId(diskid);
Bucket bt;
try {
bt = getBucket(bucketName);
} catch (SQLException ex) {
throw new ResourceNotFoundException("unable to get buckt " + bucketName + " in the system!");
}
String dskPoolId = bt.getDiskPoolId();
return dAlloc.allocDisk(dskPoolId, primary);
}
use of com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException in project ksan by infinistor.
the class ObjManager method create.
/**
* Return a new primary and replica disk mount path allocated for the path provided.
* Unless the close method is called, the allocated disk mount path is not stored permanently.
* @param bucket bucket name
* @param path the path or key of the metadata is going to be created
* @param algorithm the algorithm used to allocate osd disk
* @return return the metadata object with allocated primary and replica disk
* @throws IOException
*/
private Metadata create(String bucketName, String path, int algorithm) throws IOException, AllServiceOfflineException, ResourceNotFoundException {
Bucket bt;
logger.debug("Begin bucketName : {} key : {} alg : {} ", bucketName, path, algorithm == AllocAlgorithm.LOCALPRIMARY ? "LOCAL" : "ROUNDROBIN");
if (dAlloc == null) {
throw new ResourceNotFoundException("No disk to allocate!");
}
// create meta
try {
bt = getBucket(bucketName);
} catch (SQLException ex) {
throw new ResourceNotFoundException("Bucket(" + bucketName + ") not found due to " + ex);
}
/*if (bt == null){
throw new ResourceNotFoundException("Bucket(" + bucketName +") not exist!");
}*/
// System.out.format("create bucketName : %s path : %s diskpoolId : %s \n", bucketName, path, bt.getDiskPoolId());
logger.debug(" bucket : {} bucketId : {} diskPoolId : {} ", bt.getName(), bt.getId(), bt.getDiskPoolId());
Metadata mt = new Metadata(bucketName, path);
// allocate disk
// FIXME replace bucket id
dAlloc.allocDisk(mt, bt.getDiskPoolId(), bt.getReplicaCount(), algorithm);
logger.debug("End bucketName : {} key : {} pdiks : {} rdisk : {}", bucketName, path, mt.getPrimaryDisk().getPath(), mt.isReplicaExist() ? mt.getReplicaDisk().getPath() : "");
return mt;
}
Aggregations