use of com.qcloud.cos.model.CopyPartRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method callCOSClientWithRetry.
// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
String sdkMethod = "";
int retryIndex = 1;
int l5ErrorCodeRetryIndex = 1;
while (true) {
try {
if (request instanceof PutObjectRequest) {
sdkMethod = "putObject";
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
}
return this.cosClient.putObject((PutObjectRequest) request);
} else if (request instanceof UploadPartRequest) {
sdkMethod = "uploadPart";
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
}
return this.cosClient.uploadPart((UploadPartRequest) request);
} else if (request instanceof CopyPartRequest) {
sdkMethod = "copyPartRequest";
return this.cosClient.copyPart((CopyPartRequest) request);
} else if (request instanceof HeadBucketRequest) {
// use for checking bucket type
sdkMethod = "headBucket";
return this.cosClient.headBucket((HeadBucketRequest) request);
} else if (request instanceof RenameRequest) {
sdkMethod = "rename";
this.cosClient.rename((RenameRequest) request);
return new Object();
} else if (request instanceof GetObjectMetadataRequest) {
sdkMethod = "queryObjectMeta";
return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
} else if (request instanceof DeleteObjectRequest) {
sdkMethod = "deleteObject";
this.cosClient.deleteObject((DeleteObjectRequest) request);
return new Object();
} else if (request instanceof CopyObjectRequest) {
sdkMethod = "copyFile";
return this.cosClient.copyObject((CopyObjectRequest) request);
} else if (request instanceof GetObjectRequest) {
sdkMethod = "getObject";
return this.cosClient.getObject((GetObjectRequest) request);
} else if (request instanceof ListObjectsRequest) {
sdkMethod = "listObjects";
return this.cosClient.listObjects((ListObjectsRequest) request);
} else if (request instanceof InitiateMultipartUploadRequest) {
sdkMethod = "initiateMultipartUpload";
return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
} else if (request instanceof CompleteMultipartUploadRequest) {
sdkMethod = "completeMultipartUpload";
return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
} else if (request instanceof AbortMultipartUploadRequest) {
sdkMethod = "abortMultipartUpload";
this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
return new Object();
} else {
throw new IOException("no such method");
}
} catch (ResponseNotCompleteException nce) {
if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist must throw the exception.
handleException(nce, key);
}
LOG.warn("Complete mpu resp not complete key [{}]", key);
// todo: some other double check after cgi unified the ctime of mpu.
return new CompleteMultipartUploadResult();
} else {
throw new IOException(nce);
}
} catch (CosServiceException cse) {
String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
int statusCode = cse.getStatusCode();
String errorCode = cse.getErrorCode();
LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
// 对5xx错误进行重试
if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
// complete mpu error code might be in body when status code is 200
// double check to head object only works in big data job case which key is not same.
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
// here same like the copy request not setting the interval sleep for now
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (statusCode / 100 == 5) {
if (retryIndex <= this.maxRetryTimes) {
if (statusCode == 503) {
if (useL5Id) {
if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
// L5上报,进行重试
l5EndpointResolver.updateRouteResult(-1);
l5ErrorCodeRetryIndex = 1;
} else {
l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
}
}
}
LOG.info(errMsg, cse);
long sleepLeast = retryIndex * 300L;
long sleepBound = retryIndex * 500L;
try {
if (request instanceof PutObjectRequest) {
LOG.info("Try to reset the put object request input stream.");
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().reset();
} else {
LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
throw cse;
}
}
if (request instanceof UploadPartRequest) {
LOG.info("Try to reset the upload part request input stream.");
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().reset();
} else {
LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
throw cse;
}
}
// if direct retry may occur 403 not found the upload id.
if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
}
Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
++retryIndex;
} catch (InterruptedException e) {
throw new IOException(e.toString());
}
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else {
throw cse;
}
} catch (Exception e) {
throw new IOException(e);
}
}
}
use of com.qcloud.cos.model.CopyPartRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method uploadPartCopy.
@Override
public PartETag uploadPartCopy(String uploadId, String srcKey, String destKey, int partNum, long firstByte, long lastByte) throws IOException {
LOG.debug("Execute a part copy from the source key [{}] to the dest key [{}]. " + "upload id: {}, part number: {}, firstByte: {}, lastByte: {}.", srcKey, destKey, uploadId, partNum, firstByte, lastByte);
try {
CopyPartRequest copyPartRequest = new CopyPartRequest();
copyPartRequest.setSourceBucketName(this.bucketName);
copyPartRequest.setDestinationBucketName(this.bucketName);
copyPartRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
copyPartRequest.setUploadId(uploadId);
copyPartRequest.setSourceKey(srcKey);
copyPartRequest.setDestinationKey(destKey);
copyPartRequest.setPartNumber(partNum);
copyPartRequest.setFirstByte(firstByte);
copyPartRequest.setLastByte(lastByte);
CopyPartResult copyPartResult = (CopyPartResult) this.callCOSClientWithRetry(copyPartRequest);
return copyPartResult.getPartETag();
} catch (Exception e) {
String exceptionMessage = String.format("Copy the object part [%d-%d] from the srcKey[%s] to the destKey[%s] failed. " + "upload id: %s, part number: %d, exception: %s.", firstByte, lastByte, srcKey, destKey, uploadId, partNum, e);
handleException(new Exception(exceptionMessage), srcKey);
}
return null;
}
use of com.qcloud.cos.model.CopyPartRequest in project cos-java-sdk-v5 by tencentyun.
the class CopyCallable method copyPartsInParallel.
/**
* Submits a callable for each part to be copied to our thread pool and records its
* corresponding Future.
*/
private void copyPartsInParallel(CopyPartRequestFactory requestFactory) {
while (requestFactory.hasMoreRequests()) {
if (threadPool.isShutdown())
throw new CancellationException("TransferManager has been shutdown");
CopyPartRequest request = requestFactory.getNextCopyPartRequest();
futures.add(threadPool.submit(new CopyPartCallable(cos, request)));
}
}
use of com.qcloud.cos.model.CopyPartRequest in project cos-java-sdk-v5 by tencentyun.
the class CopyPartRequestFactory method getNextCopyPartRequest.
/**
* Constructs a copy part requests and returns it.
*
* @return Returns a new copy part request
*/
public synchronized CopyPartRequest getNextCopyPartRequest() {
final long partSize = Math.min(optimalPartSize, remainingBytes);
CopyPartRequest req = new CopyPartRequest().withSourceAppid(origReq.getSourceAppid()).withSourceBucketRegion(origReq.getSourceBucketRegion()).withSourceEndpointBuilder(origReq.getSourceEndpointBuilder()).withSourceBucketName(origReq.getSourceBucketName()).withSourceKey(origReq.getSourceKey()).withUploadId(uploadId).withPartNumber(partNumber++).withDestinationBucketName(origReq.getDestinationBucketName()).withDestinationKey(origReq.getDestinationKey()).withSourceVersionId(origReq.getSourceVersionId()).withFirstByte(Long.valueOf(offset)).withLastByte(Long.valueOf(offset + partSize - 1)).withMatchingETagConstraints(origReq.getMatchingETagConstraints()).withModifiedSinceConstraint(origReq.getModifiedSinceConstraint()).withNonmatchingETagConstraints(origReq.getNonmatchingETagConstraints()).withSourceVersionId(origReq.getSourceVersionId()).withUnmodifiedSinceConstraint(origReq.getUnmodifiedSinceConstraint()).withGeneralProgressListener(origReq.getGeneralProgressListener());
TransferManagerUtils.populateEndpointAddr(origReq, req);
offset += partSize;
remainingBytes -= partSize;
return req;
}
use of com.qcloud.cos.model.CopyPartRequest in project cos-java-sdk-v5 by tencentyun.
the class MultipartUploadDemo method copyPartUploadDemo.
// 分块copy, 表示该块的数据来自另外一个文件的某一范围, 支持跨园区, 跨bucket copy
public static void copyPartUploadDemo(String uploadId) {
// 1 初始化用户身份信息(secretId, secretKey)
COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
// 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
ClientConfig clientConfig = new ClientConfig(new Region("ap-guangzhou"));
// 3 生成cos客户端
COSClient cosclient = new COSClient(cred, clientConfig);
// bucket名需包含appid
String bucketName = "mybucket-1251668577";
String key = "aaa/bbb.txt";
CopyPartRequest copyPartRequest = new CopyPartRequest();
// 要拷贝的源文件所在的region
copyPartRequest.setSourceBucketRegion(new Region("ap-guangzhou"));
// 要拷贝的源文件的bucket名称
copyPartRequest.setSourceBucketName(bucketName);
// 要拷贝的源文件的路径
copyPartRequest.setSourceKey("aaa/ccc.txt");
// 指定要拷贝的源文件的数据范围(类似content-range)
copyPartRequest.setFirstByte(0L);
copyPartRequest.setLastByte(1048575L);
// 目的bucket名称
copyPartRequest.setDestinationBucketName(bucketName);
// 目的路径名称
copyPartRequest.setDestinationKey(key);
copyPartRequest.setPartNumber(1);
// uploadId
copyPartRequest.setUploadId(uploadId);
try {
CopyPartResult copyPartResult = cosclient.copyPart(copyPartRequest);
PartETag partETag = copyPartResult.getPartETag();
} catch (CosServiceException e) {
e.printStackTrace();
} catch (CosClientException e) {
e.printStackTrace();
}
cosclient.shutdown();
}
Aggregations