use of com.aliyun.oss.model.CompleteMultipartUploadRequest in project hadoop by apache.
the class AliyunOSSFileSystemStore method multipartUploadObject.
/**
* Upload a file as an OSS object, using multipart upload.
*
* @param key object key.
* @param file local file to upload.
* @throws IOException if failed to upload object.
*/
public void multipartUploadObject(String key, File file) throws IOException {
File object = file.getAbsoluteFile();
long dataLen = object.length();
long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
int partNum = (int) (dataLen / realPartSize);
if (dataLen % realPartSize != 0) {
partNum += 1;
}
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
ObjectMetadata meta = new ObjectMetadata();
if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
initiateMultipartUploadRequest.setObjectMetadata(meta);
InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
List<PartETag> partETags = new ArrayList<PartETag>();
String uploadId = initiateMultipartUploadResult.getUploadId();
try {
for (int i = 0; i < partNum; i++) {
// TODO: Optimize this, avoid opening the object multiple times
FileInputStream fis = new FileInputStream(object);
try {
long skipBytes = realPartSize * i;
AliyunOSSUtils.skipFully(fis, skipBytes);
long size = (realPartSize < dataLen - skipBytes) ? realPartSize : dataLen - skipBytes;
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(fis);
uploadPartRequest.setPartSize(size);
uploadPartRequest.setPartNumber(i + 1);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
statistics.incrementWriteOps(1);
partETags.add(uploadPartResult.getPartETag());
} finally {
fis.close();
}
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
LOG.debug(completeMultipartUploadResult.getETag());
} catch (OSSException | ClientException e) {
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
}
}
use of com.aliyun.oss.model.CompleteMultipartUploadRequest in project hadoop by apache.
the class AliyunOSSFileSystemStore method multipartCopy.
/**
* Use multipart copy to copy an OSS object.
* (The caller should make sure srcPath is a file and dstPath is valid)
*
* @param srcKey source key.
* @param contentLength data size of the object to copy.
* @param dstKey destination key.
* @return true if success, or false if upload is aborted.
*/
private boolean multipartCopy(String srcKey, long contentLength, String dstKey) {
long realPartSize = AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize);
int partNum = (int) (contentLength / realPartSize);
if (contentLength % realPartSize != 0) {
partNum++;
}
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, dstKey);
ObjectMetadata meta = new ObjectMetadata();
if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
initiateMultipartUploadRequest.setObjectMetadata(meta);
InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
String uploadId = initiateMultipartUploadResult.getUploadId();
List<PartETag> partETags = new ArrayList<PartETag>();
try {
for (int i = 0; i < partNum; i++) {
long skipBytes = realPartSize * i;
long size = (realPartSize < contentLength - skipBytes) ? realPartSize : contentLength - skipBytes;
UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
partCopyRequest.setSourceBucketName(bucketName);
partCopyRequest.setSourceKey(srcKey);
partCopyRequest.setBucketName(bucketName);
partCopyRequest.setKey(dstKey);
partCopyRequest.setUploadId(uploadId);
partCopyRequest.setPartSize(size);
partCopyRequest.setBeginIndex(skipBytes);
partCopyRequest.setPartNumber(i + 1);
UploadPartCopyResult partCopyResult = ossClient.uploadPartCopy(partCopyRequest);
statistics.incrementWriteOps(1);
partETags.add(partCopyResult.getPartETag());
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, dstKey, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
LOG.debug(completeMultipartUploadResult.getETag());
return true;
} catch (OSSException | ClientException e) {
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
return false;
}
}
use of com.aliyun.oss.model.CompleteMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartCopySample method main.
public static void main(String[] args) throws IOException {
/*
* Constructs a client instance with your account for accessing OSS
*/
client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
try {
/*
* Upload an object to your source bucket
*/
System.out.println("Uploading a new object to OSS from a file\n");
client.putObject(new PutObjectRequest(sourceBucketName, sourceKey, createSampleFile()));
/*
* Claim a new upload id for your target bucket
*/
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(targetBucketName, targetKey);
InitiateMultipartUploadResult initiateMultipartUploadResult = client.initiateMultipartUpload(initiateMultipartUploadRequest);
String uploadId = initiateMultipartUploadResult.getUploadId();
/*
* Calculate how many parts to be divided
*/
// 5MB
final long partSize = 5 * 1024 * 1024L;
ObjectMetadata metadata = client.getObjectMetadata(sourceBucketName, sourceKey);
long objectSize = metadata.getContentLength();
int partCount = (int) (objectSize / partSize);
if (objectSize % partSize != 0) {
partCount++;
}
if (partCount > 10000) {
throw new RuntimeException("Total parts count should not exceed 10000");
} else {
System.out.println("Total parts count " + partCount + "\n");
}
/*
* Upload multiparts by copy mode
*/
System.out.println("Begin to upload multiparts by copy mode to OSS\n");
List<PartETag> partETags = new ArrayList<PartETag>();
for (int i = 0; i < partCount; i++) {
long startPos = i * partSize;
long curPartSize = (i + 1 == partCount) ? (objectSize - startPos) : partSize;
;
UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucketName, sourceKey, targetBucketName, targetKey);
uploadPartCopyRequest.setUploadId(uploadId);
uploadPartCopyRequest.setPartSize(curPartSize);
uploadPartCopyRequest.setBeginIndex(startPos);
uploadPartCopyRequest.setPartNumber(i + 1);
UploadPartCopyResult uploadPartCopyResult = client.uploadPartCopy(uploadPartCopyRequest);
System.out.println("\tPart#" + uploadPartCopyResult.getPartNumber() + " done\n");
partETags.add(uploadPartCopyResult.getPartETag());
}
/*
* Complete to upload multiparts
*/
System.out.println("Completing to upload multiparts\n");
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucketName, targetKey, uploadId, partETags);
client.completeMultipartUpload(completeMultipartUploadRequest);
/*
* Fetch the object that newly created at the step below.
*/
System.out.println("Fetching an object");
client.getObject(new GetObjectRequest(targetBucketName, targetKey), new File(localFilePath));
} catch (OSSException oe) {
System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason.");
System.out.println("Error Message: " + oe.getErrorCode());
System.out.println("Error Code: " + oe.getErrorCode());
System.out.println("Request ID: " + oe.getRequestId());
System.out.println("Host ID: " + oe.getHostId());
} catch (ClientException ce) {
System.out.println("Caught an ClientException, which means the client encountered " + "a serious internal problem while trying to communicate with OSS, " + "such as not being able to access the network.");
System.out.println("Error Message: " + ce.getMessage());
} finally {
/*
* Do not forget to shut down the client finally to release all allocated resources.
*/
client.shutdown();
}
}
use of com.aliyun.oss.model.CompleteMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartCopyTest method testUnormalUploadPartCopyByRange.
@Test
public void testUnormalUploadPartCopyByRange() {
final String sourceBucket = "unormal-upload-part-copy-by-range-bucket-source";
final String targetBucket = "unormal-upload-part-copy-by-range-bucket-target";
final String sourceKey = "unormal-upload-part-copy-by-range-object-source";
final String targetKey = "unormal-upload-part-copy-by-range-object-target";
// 128KB
final long partSize = 128 * 1024;
try {
ossClient.createBucket(sourceBucket);
ossClient.createBucket(targetBucket);
// Put object into source bucket
final long inputStreamLength = partSize * 4;
String eTag = null;
try {
InputStream instream = genFixedLengthInputStream(inputStreamLength);
PutObjectResult result = ossClient.putObject(sourceBucket, sourceKey, instream, null);
eTag = result.getETag();
} catch (Exception e) {
Assert.fail(e.getMessage());
}
// Claim upload id for target bucket
String uploadId = claimUploadId(ossClient, targetBucket, targetKey);
// Upload part copy with invalid copy range
final int partNumber = 1;
final long beginIndex = partSize;
List<PartETag> partETags = new ArrayList<PartETag>();
UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucket, sourceKey, targetBucket, targetKey);
uploadPartCopyRequest.setPartNumber(partNumber);
uploadPartCopyRequest.setUploadId(uploadId);
uploadPartCopyRequest.setBeginIndex(beginIndex);
// Illegal copy range([beginIndex, begin + inputStreamLength]), just copy entire object
uploadPartCopyRequest.setPartSize(inputStreamLength);
UploadPartCopyResult uploadPartCopyResult = ossClient.uploadPartCopy(uploadPartCopyRequest);
partETags.add(uploadPartCopyResult.getPartETag());
Assert.assertEquals(eTag, uploadPartCopyResult.getETag());
Assert.assertEquals(partNumber, uploadPartCopyResult.getPartNumber());
Assert.assertEquals(uploadPartCopyResult.getRequestId().length(), REQUEST_ID_LEN);
ListPartsRequest listPartsRequest = new ListPartsRequest(targetBucket, targetKey, uploadId);
PartListing partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(1, partListing.getParts().size());
Assert.assertEquals(targetBucket, partListing.getBucketName());
Assert.assertEquals(targetKey, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
Assert.assertNotNull(partListing.getNextPartNumberMarker());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// Complete multipart upload
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, targetBucket, targetKey), completeMultipartUploadResult.getLocation());
Assert.assertEquals(targetBucket, completeMultipartUploadResult.getBucketName());
Assert.assertEquals(targetKey, completeMultipartUploadResult.getKey());
Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
// Get uploaded object
OSSObject o = ossClient.getObject(targetBucket, targetKey);
final long objectSize = inputStreamLength;
Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
} catch (Exception e) {
Assert.fail(e.getMessage());
} finally {
deleteBucketWithObjects(ossClient, sourceBucket);
deleteBucketWithObjects(ossClient, targetBucket);
}
}
use of com.aliyun.oss.model.CompleteMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartTest method testNormalUploadMultiparts.
@Test
public void testNormalUploadMultiparts() {
final String key = "normal-upload-multiparts-object";
// 128KB
final int partSize = 128 * 1024;
final int partCount = 10;
try {
String uploadId = claimUploadId(ossClient, bucketName, key);
// Upload parts
List<PartETag> partETags = new ArrayList<PartETag>();
for (int i = 0; i < partCount; i++) {
InputStream instream = genFixedLengthInputStream(partSize);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartNumber(i + 1);
uploadPartRequest.setPartSize(partSize);
uploadPartRequest.setUploadId(uploadId);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
Assert.assertEquals(uploadPartResult.getRequestId().length(), REQUEST_ID_LEN);
partETags.add(uploadPartResult.getPartETag());
}
// List parts
ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
PartListing partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(partCount, partListing.getParts().size());
for (int i = 0; i < partCount; i++) {
PartSummary ps = partListing.getParts().get(i);
PartETag eTag = partETags.get(i);
Assert.assertEquals(eTag.getPartNumber(), ps.getPartNumber());
Assert.assertEquals(eTag.getETag(), ps.getETag());
}
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
Assert.assertEquals(partCount, partListing.getNextPartNumberMarker().intValue());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// List single multipart upload under this bucket
ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertEquals(key, multipartUploadListing.getNextKeyMarker());
Assert.assertEquals(uploadId, multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
List<MultipartUpload> multipartUploads = multipartUploadListing.getMultipartUploads();
Assert.assertEquals(1, multipartUploads.size());
Assert.assertEquals(key, multipartUploads.get(0).getKey());
Assert.assertEquals(uploadId, multipartUploads.get(0).getUploadId());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
// Complete multipart upload
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, bucketName, key), completeMultipartUploadResult.getLocation());
Assert.assertEquals(bucketName, completeMultipartUploadResult.getBucketName());
Assert.assertEquals(key, completeMultipartUploadResult.getKey());
Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
// List single multipart uploads under this bucket again
listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(0, multipartUploadListing.getMultipartUploads().size());
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertNull(multipartUploadListing.getNextKeyMarker());
Assert.assertNull(multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
// Get uploaded object
OSSObject o = ossClient.getObject(bucketName, key);
final long objectSize = partCount * partSize;
Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
Aggregations