use of com.emc.object.s3.request.CopyPartRequest in project pravega by pravega.
the class S3FileSystemImpl method completeMultipartUpload.
@Synchronized
@Override
public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) {
Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey());
if (partMap == null) {
throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
}
try {
partMap.forEach((index, copyPart) -> {
if (copyPart.getKey() != copyPart.getSourceKey()) {
Path sourcePath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getSourceKey());
Path targetPath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getKey());
try (FileChannel sourceChannel = FileChannel.open(sourcePath, StandardOpenOption.READ);
FileChannel targetChannel = FileChannel.open(targetPath, StandardOpenOption.WRITE)) {
targetChannel.transferFrom(sourceChannel, Files.size(targetPath), copyPart.getSourceRange().getLast() + 1 - copyPart.getSourceRange().getFirst());
targetChannel.close();
AclSize aclMap = this.aclMap.get(copyPart.getKey());
this.aclMap.put(copyPart.getKey(), aclMap.withSize(Files.size(targetPath)));
} catch (IOException e) {
throw new S3Exception("NoSuchKey", 404, "NoSuchKey", "");
}
}
});
} finally {
multipartUploads.remove(request.getKey());
}
return new CompleteMultipartUploadResult();
}
use of com.emc.object.s3.request.CopyPartRequest in project pravega by pravega.
the class S3FileSystemImpl method copyPart.
@Override
public CopyPartResult copyPart(CopyPartRequest request) {
Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey());
if (partMap == null) {
throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
}
partMap.put(request.getPartNumber(), request);
CopyPartResult result = new CopyPartResult();
result.setPartNumber(request.getPartNumber());
result.setETag(request.getUploadId());
return result;
}
use of com.emc.object.s3.request.CopyPartRequest in project pravega by pravega.
the class ExtendedS3Storage method doConcat.
/**
* The concat is implemented using extended S3 implementation of multipart copy API. Please see here for
* more detail on multipart copy:
* http://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingLLJavaMPUapi.html
*
* The multipart copy is an atomic operation. We schedule two parts and commit them atomically using
* completeMultiPartUpload call. Specifically, to concatenate, we are copying the target segment T and the
* source segment S to T, so essentially we are doing T <- T + S.
*/
private Void doConcat(SegmentHandle targetHandle, long offset, String sourceSegment) throws StreamSegmentNotExistsException {
Preconditions.checkArgument(!targetHandle.isReadOnly(), "target handle must not be read-only.");
long traceId = LoggerHelpers.traceEnter(log, "concat", targetHandle.getSegmentName(), offset, sourceSegment);
SortedSet<MultipartPartETag> partEtags = new TreeSet<>();
String targetPath = config.getRoot() + targetHandle.getSegmentName();
String uploadId = client.initiateMultipartUpload(config.getBucket(), targetPath);
// check whether the target exists
if (!doExists(targetHandle.getSegmentName())) {
throw new StreamSegmentNotExistsException(targetHandle.getSegmentName());
}
// check whether the source is sealed
SegmentProperties si = doGetStreamSegmentInfo(sourceSegment);
Preconditions.checkState(si.isSealed(), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, targetHandle.getSegmentName());
// Copy the first part
CopyPartRequest copyRequest = new CopyPartRequest(config.getBucket(), targetPath, config.getBucket(), targetPath, uploadId, 1).withSourceRange(Range.fromOffsetLength(0, offset));
CopyPartResult copyResult = client.copyPart(copyRequest);
partEtags.add(new MultipartPartETag(copyResult.getPartNumber(), copyResult.getETag()));
// Copy the second part
S3ObjectMetadata metadataResult = client.getObjectMetadata(config.getBucket(), config.getRoot() + sourceSegment);
// in bytes
long objectSize = metadataResult.getContentLength();
copyRequest = new CopyPartRequest(config.getBucket(), config.getRoot() + sourceSegment, config.getBucket(), targetPath, uploadId, 2).withSourceRange(Range.fromOffsetLength(0, objectSize));
copyResult = client.copyPart(copyRequest);
partEtags.add(new MultipartPartETag(copyResult.getPartNumber(), copyResult.getETag()));
// Close the upload
client.completeMultipartUpload(new CompleteMultipartUploadRequest(config.getBucket(), targetPath, uploadId).withParts(partEtags));
client.deleteObject(config.getBucket(), config.getRoot() + sourceSegment);
LoggerHelpers.traceLeave(log, "concat", traceId);
return null;
}
Aggregations