use of com.qcloud.cos.exception.CosServiceException in project cos-java-sdk-v5 by tencentyun.
the class DefaultCosHttpClient method exeute.
@Override
public <X, Y extends CosServiceRequest> X exeute(CosHttpRequest<Y> request, HttpResponseHandler<CosServiceResponse<X>> responseHandler) throws CosClientException, CosServiceException {
HttpResponse httpResponse = null;
HttpRequestBase httpRequest = null;
bufferAndResetAbleContent(request);
// Always mark the input stream before execution.
ProgressListener progressListener = request.getProgressListener();
final InputStream originalContent = request.getContent();
if (originalContent != null) {
request.setContent(monitorStreamProgress(progressListener, originalContent));
}
if (originalContent != null && originalContent.markSupported() && !(originalContent instanceof BufferedInputStream)) {
final int readLimit = clientConfig.getReadLimit();
originalContent.mark(readLimit);
}
int retryIndex = 0;
while (true) {
try {
checkInterrupted();
if (originalContent instanceof BufferedInputStream && originalContent.markSupported()) {
// Mark everytime for BufferedInputStream, since the marker could have been
// invalidated
final int readLimit = clientConfig.getReadLimit();
originalContent.mark(readLimit);
}
// 如果是重试的则恢复流
if (retryIndex != 0 && originalContent != null) {
originalContent.reset();
}
if (retryIndex != 0) {
long delay = backoffStrategy.computeDelayBeforeNextRetry(retryIndex);
Thread.sleep(delay);
}
HttpContext context = HttpClientContext.create();
httpRequest = buildHttpRequest(request);
httpResponse = null;
httpResponse = executeOneRequest(context, httpRequest);
checkResponse(request, httpRequest, httpResponse);
break;
} catch (CosServiceException cse) {
if (cse.getStatusCode() >= 500) {
String errorMsg = String.format("failed to execute http request, due to service exception," + " httpRequest: %s, retryIdx:%d, maxErrorRetry:%d", request.toString(), retryIndex, maxErrorRetry);
log.error(errorMsg, cse);
}
closeHttpResponseStream(httpResponse);
if (!shouldRetry(request, httpResponse, cse, retryIndex, retryPolicy)) {
throw cse;
}
} catch (CosClientException cce) {
String errorMsg = String.format("failed to execute http request, due to client exception," + " httpRequest: %s, retryIdx:%d, maxErrorRetry:%d", request.toString(), retryIndex, maxErrorRetry);
log.info(errorMsg, cce);
closeHttpResponseStream(httpResponse);
if (!shouldRetry(request, httpResponse, cce, retryIndex, retryPolicy)) {
log.error(errorMsg, cce);
throw cce;
}
} catch (Exception exp) {
String errorMsg = String.format("httpClient execute occur a unknow exception, httpRequest: %s", request.toString());
closeHttpResponseStream(httpResponse);
log.error(errorMsg, exp);
throw new CosClientException(errorMsg, exp);
} finally {
++retryIndex;
}
}
try {
CosHttpResponse cosHttpResponse = createResponse(httpRequest, request, httpResponse);
return responseHandler.handle(cosHttpResponse).getResult();
} catch (Exception e) {
if (e.getMessage().equals("Premature end of chunk coded message body: closing chunk expected")) {
throw new ResponseNotCompleteException("response chunk not complete", e);
}
String errorMsg = "Unable to execute response handle: " + e.getMessage();
log.info(errorMsg, e);
CosClientException cce = new CosClientException(errorMsg, e);
throw cce;
} finally {
if (!responseHandler.needsConnectionLeftOpen()) {
httpRequest.releaseConnection();
}
}
}
use of com.qcloud.cos.exception.CosServiceException in project cloud-sdk by mizhousoft.
the class COSObjectStorageServiceImpl method doDeleteObjects.
private void doDeleteObjects(String bucketName, Collection<String> objectNames) throws CloudSDKException {
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
Set<String> names = new HashSet<>(objectNames);
// 设置要删除的key列表, 最多一次删除1000个
List<KeyVersion> keyList = new ArrayList<>();
for (String objectName : names) {
// 传入要删除的文件名
keyList.add(new KeyVersion(objectName));
}
deleteObjectsRequest.setKeys(keyList);
try {
cosClient.deleteObjects(deleteObjectsRequest);
// List<DeletedObject> deletedObjects = deleteResult.getDeletedObjects();
} catch (MultiObjectDeleteException e) {
// List<DeleteError> deleteErrors = mde.getErrors();
throw new CloudSDKException(e.getMessage(), e);
} catch (CosServiceException e) {
throw new CloudSDKException(e.getMessage(), e);
} catch (Throwable e) {
throw new CloudSDKException(e.getMessage(), e);
}
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method retrieveBlock.
@Override
public InputStream retrieveBlock(String key, long byteRangeStart, long byteRangeEnd) throws IOException {
LOG.debug("Retrieve the cos key: {}, byte range start: {}, byte range end: {}.", key, byteRangeStart, byteRangeEnd);
try {
GetObjectRequest request = new GetObjectRequest(this.bucketName, key);
request.setRange(byteRangeStart, byteRangeEnd);
if (this.trafficLimit >= 0) {
request.setTrafficLimit(this.trafficLimit);
}
this.setEncryptionMetadata(request, new ObjectMetadata());
COSObject cosObject = (COSObject) this.callCOSClientWithRetry(request);
return cosObject.getObjectContent();
} catch (CosServiceException e) {
String errMsg = String.format("Retrieving the key %s with the byteRangeStart [%d] " + "occurs an exception: %s.", key, byteRangeStart, e);
handleException(new Exception(errMsg), key);
} catch (CosClientException e) {
String errMsg = String.format("Retrieving key %s with the byteRangeStart [%d] and the byteRangeEnd [%d] " + "occurs an exception: %s.", key, byteRangeStart, byteRangeEnd, e);
handleException(new Exception(errMsg), key);
}
return null;
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method storeEmptyFile.
// for cos, storeEmptyFile means create a directory
@Override
public void storeEmptyFile(String key) throws IOException {
LOG.debug("Store an empty file to the key: {}.", key);
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(0);
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
InputStream input = new ByteArrayInputStream(new byte[0]);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, input, objectMetadata);
if (null != this.storageClass) {
putObjectRequest.setStorageClass(this.storageClass);
}
try {
PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
LOG.debug("Store the empty file successfully, cos key: {}, ETag: {}.", key, putObjectResult.getETag());
} catch (CosServiceException cse) {
int statusCode = cse.getStatusCode();
if (statusCode == 409) {
// 并发上传文件导致,再check一遍文件是否存在
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// 文件还是不存在,必须要抛出异常
handleException(cse, key);
}
LOG.warn("Upload the file [{}] concurrently.", key);
} else {
// 其他错误必须抛出
handleException(cse, key);
}
} catch (Exception e) {
String errMsg = String.format("Store the empty file failed, cos key: %s, " + "exception: %s", key, e);
handleException(new Exception(errMsg), key);
}
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method callCOSClientWithRetry.
// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
String sdkMethod = "";
int retryIndex = 1;
int l5ErrorCodeRetryIndex = 1;
while (true) {
try {
if (request instanceof PutObjectRequest) {
sdkMethod = "putObject";
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
}
return this.cosClient.putObject((PutObjectRequest) request);
} else if (request instanceof UploadPartRequest) {
sdkMethod = "uploadPart";
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
}
return this.cosClient.uploadPart((UploadPartRequest) request);
} else if (request instanceof CopyPartRequest) {
sdkMethod = "copyPartRequest";
return this.cosClient.copyPart((CopyPartRequest) request);
} else if (request instanceof HeadBucketRequest) {
// use for checking bucket type
sdkMethod = "headBucket";
return this.cosClient.headBucket((HeadBucketRequest) request);
} else if (request instanceof RenameRequest) {
sdkMethod = "rename";
this.cosClient.rename((RenameRequest) request);
return new Object();
} else if (request instanceof GetObjectMetadataRequest) {
sdkMethod = "queryObjectMeta";
return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
} else if (request instanceof DeleteObjectRequest) {
sdkMethod = "deleteObject";
this.cosClient.deleteObject((DeleteObjectRequest) request);
return new Object();
} else if (request instanceof CopyObjectRequest) {
sdkMethod = "copyFile";
return this.cosClient.copyObject((CopyObjectRequest) request);
} else if (request instanceof GetObjectRequest) {
sdkMethod = "getObject";
return this.cosClient.getObject((GetObjectRequest) request);
} else if (request instanceof ListObjectsRequest) {
sdkMethod = "listObjects";
return this.cosClient.listObjects((ListObjectsRequest) request);
} else if (request instanceof InitiateMultipartUploadRequest) {
sdkMethod = "initiateMultipartUpload";
return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
} else if (request instanceof CompleteMultipartUploadRequest) {
sdkMethod = "completeMultipartUpload";
return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
} else if (request instanceof AbortMultipartUploadRequest) {
sdkMethod = "abortMultipartUpload";
this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
return new Object();
} else {
throw new IOException("no such method");
}
} catch (ResponseNotCompleteException nce) {
if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist must throw the exception.
handleException(nce, key);
}
LOG.warn("Complete mpu resp not complete key [{}]", key);
// todo: some other double check after cgi unified the ctime of mpu.
return new CompleteMultipartUploadResult();
} else {
throw new IOException(nce);
}
} catch (CosServiceException cse) {
String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
int statusCode = cse.getStatusCode();
String errorCode = cse.getErrorCode();
LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
// 对5xx错误进行重试
if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
// complete mpu error code might be in body when status code is 200
// double check to head object only works in big data job case which key is not same.
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
// here same like the copy request not setting the interval sleep for now
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (statusCode / 100 == 5) {
if (retryIndex <= this.maxRetryTimes) {
if (statusCode == 503) {
if (useL5Id) {
if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
// L5上报,进行重试
l5EndpointResolver.updateRouteResult(-1);
l5ErrorCodeRetryIndex = 1;
} else {
l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
}
}
}
LOG.info(errMsg, cse);
long sleepLeast = retryIndex * 300L;
long sleepBound = retryIndex * 500L;
try {
if (request instanceof PutObjectRequest) {
LOG.info("Try to reset the put object request input stream.");
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().reset();
} else {
LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
throw cse;
}
}
if (request instanceof UploadPartRequest) {
LOG.info("Try to reset the upload part request input stream.");
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().reset();
} else {
LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
throw cse;
}
}
// if direct retry may occur 403 not found the upload id.
if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
}
Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
++retryIndex;
} catch (InterruptedException e) {
throw new IOException(e.toString());
}
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else {
throw cse;
}
} catch (Exception e) {
throw new IOException(e);
}
}
}
Aggregations