use of com.qcloud.cos.model.CompleteMultipartUploadResult in project cos-java-sdk-v5 by tencentyun.
the class BlindWatermarkDemo method addBlindWatermarkWithMultipart.
public static void addBlindWatermarkWithMultipart(COSClient cosClient) throws FileNotFoundException {
// bucket名需包含appid
// api 请参考:https://cloud.tencent.com/document/product/436/46782
String bucketName = "examplebucket-1250000000";
String key = "qrcode.png";
File localFile = new File("E://qrcode.png");
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
InitiateMultipartUploadResult initResult = cosClient.initiateMultipartUpload(request);
String uploadId = initResult.getUploadId();
// 上传分块
List<PartETag> partETags = new LinkedList<>();
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setUploadId(uploadId);
// 设置分块的数据来源输入流
uploadPartRequest.setInputStream(new FileInputStream(localFile));
// 设置分块的长度
// 设置数据长度
uploadPartRequest.setPartSize(localFile.length());
// 假设要上传的part编号是10
uploadPartRequest.setPartNumber(1);
UploadPartResult uploadPartResult = cosClient.uploadPart(uploadPartRequest);
PartETag partETag = uploadPartResult.getPartETag();
partETags.add(partETag);
// 合并分块并带上图像处理参数
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
PicOperations picOperations = new PicOperations();
picOperations.setIsPicInfo(1);
List<PicOperations.Rule> ruleList = new LinkedList<>();
PicOperations.Rule rule1 = new PicOperations.Rule();
rule1.setBucket(bucketName);
rule1.setFileId("qrcode-watermark.png");
rule1.setRule("watermark/3/type/3/text/dGVuY2VudCBjbG91ZA==");
ruleList.add(rule1);
picOperations.setRules(ruleList);
completeMultipartUploadRequest.setPicOperations(picOperations);
CompleteMultipartUploadResult completeMultipartUploadResult = cosClient.completeMultipartUpload(completeMultipartUploadRequest);
CIUploadResult ciUploadResult = completeMultipartUploadResult.getCiUploadResult();
System.out.println(completeMultipartUploadResult.getRequestId());
System.out.println(ciUploadResult.getOriginalInfo().getEtag());
for (CIObject ciObject : ciUploadResult.getProcessResults().getObjectList()) {
System.out.println(ciObject.getLocation());
}
}
use of com.qcloud.cos.model.CompleteMultipartUploadResult in project cos-java-sdk-v5 by tencentyun.
the class COSCryptoModuleBase method completeMultipartUploadSecurely.
@Override
public CompleteMultipartUploadResult completeMultipartUploadSecurely(CompleteMultipartUploadRequest req) {
String uploadId = req.getUploadId();
final MultipartUploadCryptoContext uploadContext = multipartUploadContexts.get(uploadId);
if (uploadContext != null && !uploadContext.hasFinalPartBeenSeen()) {
throw new CosClientException("Unable to complete an encrypted multipart upload without being told which part was the last. " + "Without knowing which part was the last, the encrypted data in COS is incomplete and corrupt.");
}
CompleteMultipartUploadResult result = cos.completeMultipartUpload(req);
// after the whole upload has completed correctly.
if (uploadContext != null && cryptoConfig.getStorageMode() == InstructionFile) {
// Put the instruction file into COS
cos.putObject(createInstructionPutRequest(uploadContext.getBucketName(), uploadContext.getKey(), uploadContext.getContentCryptoMaterial()));
}
multipartUploadContexts.remove(uploadId);
return result;
}
use of com.qcloud.cos.model.CompleteMultipartUploadResult in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method callCOSClientWithRetry.
// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
String sdkMethod = "";
int retryIndex = 1;
int l5ErrorCodeRetryIndex = 1;
while (true) {
try {
if (request instanceof PutObjectRequest) {
sdkMethod = "putObject";
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
}
return this.cosClient.putObject((PutObjectRequest) request);
} else if (request instanceof UploadPartRequest) {
sdkMethod = "uploadPart";
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
}
return this.cosClient.uploadPart((UploadPartRequest) request);
} else if (request instanceof CopyPartRequest) {
sdkMethod = "copyPartRequest";
return this.cosClient.copyPart((CopyPartRequest) request);
} else if (request instanceof HeadBucketRequest) {
// use for checking bucket type
sdkMethod = "headBucket";
return this.cosClient.headBucket((HeadBucketRequest) request);
} else if (request instanceof RenameRequest) {
sdkMethod = "rename";
this.cosClient.rename((RenameRequest) request);
return new Object();
} else if (request instanceof GetObjectMetadataRequest) {
sdkMethod = "queryObjectMeta";
return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
} else if (request instanceof DeleteObjectRequest) {
sdkMethod = "deleteObject";
this.cosClient.deleteObject((DeleteObjectRequest) request);
return new Object();
} else if (request instanceof CopyObjectRequest) {
sdkMethod = "copyFile";
return this.cosClient.copyObject((CopyObjectRequest) request);
} else if (request instanceof GetObjectRequest) {
sdkMethod = "getObject";
return this.cosClient.getObject((GetObjectRequest) request);
} else if (request instanceof ListObjectsRequest) {
sdkMethod = "listObjects";
return this.cosClient.listObjects((ListObjectsRequest) request);
} else if (request instanceof InitiateMultipartUploadRequest) {
sdkMethod = "initiateMultipartUpload";
return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
} else if (request instanceof CompleteMultipartUploadRequest) {
sdkMethod = "completeMultipartUpload";
return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
} else if (request instanceof AbortMultipartUploadRequest) {
sdkMethod = "abortMultipartUpload";
this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
return new Object();
} else {
throw new IOException("no such method");
}
} catch (ResponseNotCompleteException nce) {
if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist must throw the exception.
handleException(nce, key);
}
LOG.warn("Complete mpu resp not complete key [{}]", key);
// todo: some other double check after cgi unified the ctime of mpu.
return new CompleteMultipartUploadResult();
} else {
throw new IOException(nce);
}
} catch (CosServiceException cse) {
String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
int statusCode = cse.getStatusCode();
String errorCode = cse.getErrorCode();
LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
// 对5xx错误进行重试
if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
// complete mpu error code might be in body when status code is 200
// double check to head object only works in big data job case which key is not same.
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
// here same like the copy request not setting the interval sleep for now
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (statusCode / 100 == 5) {
if (retryIndex <= this.maxRetryTimes) {
if (statusCode == 503) {
if (useL5Id) {
if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
// L5上报,进行重试
l5EndpointResolver.updateRouteResult(-1);
l5ErrorCodeRetryIndex = 1;
} else {
l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
}
}
}
LOG.info(errMsg, cse);
long sleepLeast = retryIndex * 300L;
long sleepBound = retryIndex * 500L;
try {
if (request instanceof PutObjectRequest) {
LOG.info("Try to reset the put object request input stream.");
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().reset();
} else {
LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
throw cse;
}
}
if (request instanceof UploadPartRequest) {
LOG.info("Try to reset the upload part request input stream.");
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().reset();
} else {
LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
throw cse;
}
}
// if direct retry may occur 403 not found the upload id.
if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
}
Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
++retryIndex;
} catch (InterruptedException e) {
throw new IOException(e.toString());
}
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else {
throw cse;
}
} catch (Exception e) {
throw new IOException(e);
}
}
}
use of com.qcloud.cos.model.CompleteMultipartUploadResult in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method completeMultipartUpload.
/**
* complete cos mpu, sometimes return null complete mpu result
* @param key cos key
* @param uploadId upload id
* @param partETagList each part etag list
* @return result
* @throws IOException when fail to complete the multipart upload.
*/
public CompleteMultipartUploadResult completeMultipartUpload(String key, String uploadId, List<PartETag> partETagList) throws IOException {
Collections.sort(partETagList, new Comparator<PartETag>() {
@Override
public int compare(PartETag o1, PartETag o2) {
return o1.getPartNumber() - o2.getPartNumber();
}
});
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETagList);
completeMultipartUploadRequest.setObjectMetadata(objectMetadata);
return (CompleteMultipartUploadResult) this.callCOSClientWithRetry(completeMultipartUploadRequest);
} catch (CosServiceException cse) {
// when first calling with 503 access time out, next retry will 409.
int statusCode = cse.getStatusCode();
if (statusCode == 409) {
// check file whether exist
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist through exception
handleException(cse, key);
}
LOG.warn("Upload the cos key [{}] complete mpu concurrently", key);
} else {
// other exception
String errMsg = String.format("Complete the multipart upload failed. " + "cos service exception, cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, cse.toString());
handleException(new Exception(errMsg), key);
}
} catch (Exception e) {
String errMsg = String.format("Complete the multipart upload failed. " + "cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, e.toString());
handleException(new Exception(errMsg), key);
}
return null;
}
use of com.qcloud.cos.model.CompleteMultipartUploadResult in project cos-java-sdk-v5 by tencentyun.
the class CompleteMultipartUpload method call.
@Override
public UploadResult call() throws Exception {
CompleteMultipartUploadResult res;
try {
CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest(origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()).withGeneralProgressListener(origReq.getGeneralProgressListener());
ObjectMetadata origMeta = origReq.getMetadata();
if (origMeta != null) {
ObjectMetadata objMeta = req.getObjectMetadata();
if (objMeta == null) {
objMeta = new ObjectMetadata();
}
objMeta.setUserMetadata(origMeta.getUserMetadata());
req.setObjectMetadata(objMeta);
}
if (origReq.getPicOperations() != null) {
req.setPicOperations(origReq.getPicOperations());
}
TransferManagerUtils.populateEndpointAddr(origReq, req);
res = cos.completeMultipartUpload(req);
} catch (Exception e) {
monitor.uploadFailed();
throw e;
}
UploadResult uploadResult = new UploadResult();
uploadResult.setBucketName(origReq.getBucketName());
uploadResult.setKey(origReq.getKey());
uploadResult.setETag(res.getETag());
uploadResult.setVersionId(res.getVersionId());
uploadResult.setRequestId(res.getRequestId());
uploadResult.setDateStr(res.getDateStr());
uploadResult.setCrc64Ecma(res.getCrc64Ecma());
uploadResult.setCiUploadResult(res.getCiUploadResult());
monitor.uploadComplete();
return uploadResult;
}
Aggregations