use of com.aliyun.oss.model.UploadPartRequest in project hadoop by apache.
the class AliyunOSSFileSystemStore method multipartUploadObject.
/**
* Upload a file as an OSS object, using multipart upload.
*
* @param key object key.
* @param file local file to upload.
* @throws IOException if failed to upload object.
*/
public void multipartUploadObject(String key, File file) throws IOException {
File object = file.getAbsoluteFile();
long dataLen = object.length();
long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
int partNum = (int) (dataLen / realPartSize);
if (dataLen % realPartSize != 0) {
partNum += 1;
}
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
ObjectMetadata meta = new ObjectMetadata();
if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
}
initiateMultipartUploadRequest.setObjectMetadata(meta);
InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
List<PartETag> partETags = new ArrayList<PartETag>();
String uploadId = initiateMultipartUploadResult.getUploadId();
try {
for (int i = 0; i < partNum; i++) {
// TODO: Optimize this, avoid opening the object multiple times
FileInputStream fis = new FileInputStream(object);
try {
long skipBytes = realPartSize * i;
AliyunOSSUtils.skipFully(fis, skipBytes);
long size = (realPartSize < dataLen - skipBytes) ? realPartSize : dataLen - skipBytes;
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(fis);
uploadPartRequest.setPartSize(size);
uploadPartRequest.setPartNumber(i + 1);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
statistics.incrementWriteOps(1);
partETags.add(uploadPartResult.getPartETag());
} finally {
fis.close();
}
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
LOG.debug(completeMultipartUploadResult.getETag());
} catch (OSSException | ClientException e) {
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
}
}
use of com.aliyun.oss.model.UploadPartRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartTest method testNormalUploadMultiparts.
@Test
public void testNormalUploadMultiparts() {
final String key = "normal-upload-multiparts-object";
// 128KB
final int partSize = 128 * 1024;
final int partCount = 10;
try {
String uploadId = claimUploadId(ossClient, bucketName, key);
// Upload parts
List<PartETag> partETags = new ArrayList<PartETag>();
for (int i = 0; i < partCount; i++) {
InputStream instream = genFixedLengthInputStream(partSize);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartNumber(i + 1);
uploadPartRequest.setPartSize(partSize);
uploadPartRequest.setUploadId(uploadId);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
Assert.assertEquals(uploadPartResult.getRequestId().length(), REQUEST_ID_LEN);
partETags.add(uploadPartResult.getPartETag());
}
// List parts
ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
PartListing partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(partCount, partListing.getParts().size());
for (int i = 0; i < partCount; i++) {
PartSummary ps = partListing.getParts().get(i);
PartETag eTag = partETags.get(i);
Assert.assertEquals(eTag.getPartNumber(), ps.getPartNumber());
Assert.assertEquals(eTag.getETag(), ps.getETag());
}
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
Assert.assertEquals(partCount, partListing.getNextPartNumberMarker().intValue());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// List single multipart upload under this bucket
ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertEquals(key, multipartUploadListing.getNextKeyMarker());
Assert.assertEquals(uploadId, multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
List<MultipartUpload> multipartUploads = multipartUploadListing.getMultipartUploads();
Assert.assertEquals(1, multipartUploads.size());
Assert.assertEquals(key, multipartUploads.get(0).getKey());
Assert.assertEquals(uploadId, multipartUploads.get(0).getUploadId());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
// Complete multipart upload
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, bucketName, key), completeMultipartUploadResult.getLocation());
Assert.assertEquals(bucketName, completeMultipartUploadResult.getBucketName());
Assert.assertEquals(key, completeMultipartUploadResult.getKey());
Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
// List single multipart uploads under this bucket again
listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(0, multipartUploadListing.getMultipartUploads().size());
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertNull(multipartUploadListing.getNextKeyMarker());
Assert.assertNull(multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
// Get uploaded object
OSSObject o = ossClient.getObject(bucketName, key);
final long objectSize = partCount * partSize;
Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
use of com.aliyun.oss.model.UploadPartRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartTest method testNormalListMultipartUploadsWithEncoding.
@Test
public void testNormalListMultipartUploadsWithEncoding() {
try {
// Add LIST_UPLOAD_MAX_RETURNS + 1 + lv2KeyCount objects to bucket
List<String> existingKeys = new ArrayList<String>();
final int lv2KeyCount = 11;
final int multipartUploadCount = LIST_UPLOAD_MAX_RETURNS + 1 + lv2KeyCount;
final String lv0KeyPrefix = "常记溪亭日暮,沉醉不知归路。";
final String lv1KeyPrefix = "常记溪亭日暮,沉醉不知归路。/昨夜雨疏风骤,浓睡不消残酒。-";
final String lv2KeyPrefix = "常记溪亭日暮,沉醉不知归路。/昨夜雨疏风骤,浓睡不消残酒。/湖上风来波浩渺,秋已暮、红稀香少。-";
for (int i = 0; i <= LIST_UPLOAD_MAX_RETURNS; i++) {
if (i % 10 != 0) {
existingKeys.add(lv0KeyPrefix + i);
} else {
existingKeys.add(lv1KeyPrefix + i);
if (i % 100 == 0) {
existingKeys.add(lv2KeyPrefix + i);
}
}
}
// Upload single part for each multipart upload
// 128B
final int partSize = 128;
List<String> uploadIds = new ArrayList<String>(multipartUploadCount);
for (int i = 0; i < multipartUploadCount; i++) {
String key = existingKeys.get(i);
String uploadId = claimUploadId(ossClient, bucketName, key);
uploadIds.add(uploadId);
InputStream instream = genFixedLengthInputStream(partSize);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartNumber(1);
uploadPartRequest.setPartSize(partSize);
uploadPartRequest.setUploadId(uploadId);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
Assert.assertEquals(1, uploadPartResult.getPartNumber());
Assert.assertEquals(uploadPartResult.getRequestId().length(), REQUEST_ID_LEN);
}
// List multipart uploads without any conditions
ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
listMultipartUploadsRequest.setEncodingType(DEFAULT_ENCODING_TYPE);
MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_UPLOAD_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertTrue(multipartUploadListing.isTruncated());
Assert.assertNotNull(multipartUploadListing.getNextKeyMarker());
Assert.assertNotNull(multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
List<MultipartUpload> multipartUploads = multipartUploadListing.getMultipartUploads();
Assert.assertEquals(LIST_UPLOAD_MAX_RETURNS, multipartUploads.size());
for (int i = 0; i < LIST_UPLOAD_MAX_RETURNS; i++) {
Assert.assertTrue(existingKeys.contains(URLDecoder.decode(multipartUploads.get(i).getKey(), "UTF-8")));
Assert.assertTrue(uploadIds.contains(multipartUploads.get(i).getUploadId()));
}
// Abort all incompleted multipart uploads
for (int i = 0; i < multipartUploadCount; i++) {
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, existingKeys.get(i), uploadIds.get(i));
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
}
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
use of com.aliyun.oss.model.UploadPartRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartTest method testNormalUploadSinglePart.
@Test
public void testNormalUploadSinglePart() {
final String key = "normal-upload-single-part-object";
// 128KB
final int partSize = 128 * 1024;
try {
String uploadId = claimUploadId(ossClient, bucketName, key);
InputStream instream = genFixedLengthInputStream(partSize);
// Upload single part
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartNumber(2);
uploadPartRequest.setPartSize(partSize);
uploadPartRequest.setUploadId(uploadId);
ossClient.uploadPart(uploadPartRequest);
// List single multipart upload under this bucket
ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertEquals(key, multipartUploadListing.getNextKeyMarker());
Assert.assertEquals(uploadId, multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
List<MultipartUpload> multipartUploads = multipartUploadListing.getMultipartUploads();
Assert.assertEquals(1, multipartUploads.size());
Assert.assertEquals(key, multipartUploads.get(0).getKey());
Assert.assertEquals(uploadId, multipartUploads.get(0).getUploadId());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
// Abort multipart upload
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId);
ossClient.abortMultipartUpload(abortMultipartUploadRequest);
// List single multipart upload under this bucket again
listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
Assert.assertEquals(0, multipartUploadListing.getMultipartUploads().size());
Assert.assertEquals(bucketName, multipartUploadListing.getBucketName());
Assert.assertEquals(LIST_PART_MAX_RETURNS, multipartUploadListing.getMaxUploads());
Assert.assertFalse(multipartUploadListing.isTruncated());
Assert.assertNull(multipartUploadListing.getNextKeyMarker());
Assert.assertNull(multipartUploadListing.getNextUploadIdMarker());
Assert.assertNull(multipartUploadListing.getDelimiter());
Assert.assertNull(multipartUploadListing.getPrefix());
Assert.assertNull(multipartUploadListing.getKeyMarker());
Assert.assertNull(multipartUploadListing.getUploadIdMarker());
Assert.assertEquals(multipartUploadListing.getRequestId().length(), REQUEST_ID_LEN);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
use of com.aliyun.oss.model.UploadPartRequest in project aliyun-oss-java-sdk by aliyun.
the class UploadPartTest method testNormalListParts.
@Test
public void testNormalListParts() {
final String key = "normal-list-parts-object";
// 128KB
final int partSize = 128 * 1024;
final int partCount = 25;
try {
String uploadId = claimUploadId(ossClient, bucketName, key);
// List parts under empty bucket
ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
PartListing partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(0, partListing.getParts().size());
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
Assert.assertNull(partListing.getNextPartNumberMarker());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// Upload parts
List<PartETag> partETags = new ArrayList<PartETag>();
for (int i = 0; i < partCount; i++) {
InputStream instream = genFixedLengthInputStream(partSize);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setInputStream(instream);
uploadPartRequest.setPartNumber(i + 1);
uploadPartRequest.setPartSize(partSize);
uploadPartRequest.setUploadId(uploadId);
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
Assert.assertEquals(uploadPartResult.getRequestId().length(), REQUEST_ID_LEN);
partETags.add(uploadPartResult.getPartETag());
}
// List parts without any special conditions
listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(partCount, partListing.getParts().size());
for (int i = 0; i < partCount; i++) {
PartSummary ps = partListing.getParts().get(i);
PartETag eTag = partETags.get(i);
Assert.assertEquals(eTag.getPartNumber(), ps.getPartNumber());
Assert.assertEquals(eTag.getETag(), ps.getETag());
}
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
Assert.assertEquals(partCount, partListing.getNextPartNumberMarker().intValue());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// List 'max-parts' parts each time
final int maxParts = 15;
listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
listPartsRequest.setMaxParts(maxParts);
partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(maxParts, partListing.getParts().size());
for (int i = 0; i < maxParts; i++) {
PartSummary ps = partListing.getParts().get(i);
PartETag eTag = partETags.get(i);
Assert.assertEquals(eTag.getPartNumber(), ps.getPartNumber());
Assert.assertEquals(eTag.getETag(), ps.getETag());
Assert.assertEquals(partSize, ps.getSize());
}
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(maxParts, partListing.getMaxParts().intValue());
Assert.assertEquals(maxParts, partListing.getNextPartNumberMarker().intValue());
Assert.assertTrue(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// List 'max-parts' parts with 'part-number-marker'
final int partNumberMarker = 20;
listPartsRequest.setPartNumberMarker(partNumberMarker);
partListing = ossClient.listParts(listPartsRequest);
Assert.assertEquals(partCount - partNumberMarker, partListing.getParts().size());
for (int i = 0; i < (partCount - partNumberMarker); i++) {
PartSummary ps = partListing.getParts().get(i);
PartETag eTag = partETags.get(partNumberMarker + i);
Assert.assertEquals(eTag.getPartNumber(), ps.getPartNumber());
Assert.assertEquals(eTag.getETag(), ps.getETag());
Assert.assertEquals(partSize, ps.getSize());
}
Assert.assertEquals(bucketName, partListing.getBucketName());
Assert.assertEquals(key, partListing.getKey());
Assert.assertEquals(uploadId, partListing.getUploadId());
Assert.assertEquals(maxParts, partListing.getMaxParts().intValue());
Assert.assertEquals(partCount, partListing.getNextPartNumberMarker().intValue());
Assert.assertFalse(partListing.isTruncated());
Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
// Complete multipart upload
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, bucketName, key), completeMultipartUploadResult.getLocation());
Assert.assertEquals(bucketName, completeMultipartUploadResult.getBucketName());
Assert.assertEquals(key, completeMultipartUploadResult.getKey());
Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
// Get uploaded object
OSSObject o = ossClient.getObject(bucketName, key);
final long objectSize = partCount * partSize;
Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
Aggregations